{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HopperBulletEnv with Twin Delayed DDPG (TD3)\n",
    "\n",
    "### Paper: https://arxiv.org/abs/1802.09477\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Start the Environment and Agent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_dir=D:\\Users\\user\\Anaconda2_02Aug2019\\envs\\ml2\\lib\\site-packages\\pybullet_envs\\bullet\n",
      "device:  cpu\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import pybullet_envs\n",
    "import pybullet_envs.bullet as bul\n",
    "\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import gym\n",
    "import os\n",
    "\n",
    "import time\n",
    "from TwinDelayed import Actor, Critic, ReplayBuffer, TD3\n",
    "from collections import deque\n",
    "import itertools as it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_dim:  15 , action_dim:  3\n",
      "n max_action:  1.0 , threshold:  2500.0 , std_noise:  0.02\n"
     ]
    }
   ],
   "source": [
    "start_timestep=1e4\n",
    "\n",
    "std_noise=0.02\n",
    "\n",
    "env = gym.make('HopperBulletEnv-v0')\n",
    "\n",
    "# Set seeds\n",
    "seed = 12345\n",
    "env.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "state = env.reset()\n",
    "state_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0] \n",
    "max_action = float(env.action_space.high[0])\n",
    "threshold = env.spec.reward_threshold\n",
    "\n",
    "print('start_dim: ', state_dim, ', action_dim: ', action_dim)\n",
    "print('n max_action: ', max_action, ', threshold: ', threshold, ', std_noise: ', std_noise)\n",
    "\n",
    "agent = TD3(state_dim, action_dim, max_action)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Traning the agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save(agent = agent, filename='checkpnt, directory = 'dir_chkpoint')     \n",
    "def save(agent, filename, directory):\n",
    "    torch.save(agent.actor.state_dict(), '%s/%s_actor.pth' % (directory, filename))\n",
    "    torch.save(agent.critic.state_dict(), '%s/%s_critic.pth' % (directory, filename))\n",
    "    torch.save(agent.actor_target.state_dict(), '%s/%s_actor_t.pth' % (directory, filename))\n",
    "    torch.save(agent.critic_target.state_dict(), '%s/%s_critic_t.pth' % (directory, filename))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Low in action space:  [-1. -1. -1.] , High:  [1. 1. 1.] , Action_dim:  3\n",
      "Ep. 10, Timestep 118,  Ep.Timesteps 10, Score: 20.26, Avg.Score: 21.03, Max.Score: 25.77, Time: 00:00:05 \n",
      "Ep. 20, Timestep 238,  Ep.Timesteps 11, Score: 19.57, Avg.Score: 20.25, Max.Score: 26.70, Time: 00:00:11 \n",
      "Ep. 30, Timestep 345,  Ep.Timesteps 15, Score: 18.58, Avg.Score: 20.02, Max.Score: 26.70, Time: 00:00:15 \n",
      "Ep. 40, Timestep 469,  Ep.Timesteps 8, Score: 19.57, Avg.Score: 20.03, Max.Score: 26.70, Time: 00:00:19 \n",
      "Ep. 50, Timestep 599,  Ep.Timesteps 16, Score: 23.34, Avg.Score: 20.01, Max.Score: 26.70, Time: 00:00:24 \n",
      "Ep. 60, Timestep 708,  Ep.Timesteps 20, Score: 24.28, Avg.Score: 19.86, Max.Score: 26.70, Time: 00:00:29 \n",
      "Ep. 70, Timestep 817,  Ep.Timesteps 7, Score: 18.65, Avg.Score: 19.93, Max.Score: 26.70, Time: 00:00:34 \n",
      "Ep. 80, Timestep 964,  Ep.Timesteps 20, Score: 22.03, Avg.Score: 20.17, Max.Score: 28.56, Time: 00:00:43 \n",
      "Ep. 90, Timestep 1078,  Ep.Timesteps 11, Score: 19.33, Avg.Score: 20.03, Max.Score: 28.56, Time: 00:00:52 \n",
      "Ep. 100, Timestep 1180,  Ep.Timesteps 5, Score: 16.82, Avg.Score: 20.05, Max.Score: 28.56, Time: 00:01:00 \n",
      "Ep. 110, Timestep 1273,  Ep.Timesteps 7, Score: 19.67, Avg.Score: 19.76, Max.Score: 28.56, Time: 00:01:05 \n",
      "Ep. 120, Timestep 1379,  Ep.Timesteps 11, Score: 24.60, Avg.Score: 19.93, Max.Score: 28.56, Time: 00:01:08 \n",
      "Ep. 130, Timestep 1497,  Ep.Timesteps 6, Score: 18.60, Avg.Score: 19.99, Max.Score: 28.56, Time: 00:01:16 \n",
      "Ep. 140, Timestep 1639,  Ep.Timesteps 9, Score: 19.28, Avg.Score: 19.99, Max.Score: 33.29, Time: 00:01:24 \n",
      "Ep. 150, Timestep 1732,  Ep.Timesteps 8, Score: 21.44, Avg.Score: 20.05, Max.Score: 33.29, Time: 00:01:29 \n",
      "Ep. 160, Timestep 1845,  Ep.Timesteps 10, Score: 20.60, Avg.Score: 20.10, Max.Score: 33.29, Time: 00:01:35 \n",
      "Ep. 170, Timestep 1956,  Ep.Timesteps 15, Score: 19.69, Avg.Score: 20.05, Max.Score: 33.29, Time: 00:01:41 \n",
      "Ep. 180, Timestep 2091,  Ep.Timesteps 6, Score: 18.19, Avg.Score: 19.81, Max.Score: 33.29, Time: 00:01:51 \n",
      "Ep. 190, Timestep 2215,  Ep.Timesteps 21, Score: 24.94, Avg.Score: 19.98, Max.Score: 33.29, Time: 00:01:54 \n",
      "Ep. 200, Timestep 2349,  Ep.Timesteps 14, Score: 22.27, Avg.Score: 19.95, Max.Score: 33.29, Time: 00:02:02 \n",
      "Ep. 210, Timestep 2475,  Ep.Timesteps 13, Score: 21.67, Avg.Score: 20.26, Max.Score: 33.29, Time: 00:02:09 \n",
      "Ep. 220, Timestep 2606,  Ep.Timesteps 13, Score: 21.33, Avg.Score: 20.23, Max.Score: 33.29, Time: 00:02:17 \n",
      "Ep. 230, Timestep 2735,  Ep.Timesteps 7, Score: 19.76, Avg.Score: 20.26, Max.Score: 33.29, Time: 00:02:26 \n",
      "Ep. 240, Timestep 2836,  Ep.Timesteps 14, Score: 23.52, Avg.Score: 20.26, Max.Score: 29.39, Time: 00:02:31 \n",
      "Ep. 250, Timestep 2930,  Ep.Timesteps 19, Score: 20.28, Avg.Score: 20.06, Max.Score: 29.39, Time: 00:02:36 \n",
      "Ep. 260, Timestep 3035,  Ep.Timesteps 7, Score: 19.47, Avg.Score: 20.05, Max.Score: 29.39, Time: 00:02:41 \n",
      "Ep. 270, Timestep 3138,  Ep.Timesteps 13, Score: 19.79, Avg.Score: 20.11, Max.Score: 29.39, Time: 00:02:48 \n",
      "Ep. 280, Timestep 3251,  Ep.Timesteps 9, Score: 19.47, Avg.Score: 19.99, Max.Score: 29.39, Time: 00:02:55 \n",
      "Ep. 290, Timestep 3379,  Ep.Timesteps 15, Score: 21.22, Avg.Score: 20.05, Max.Score: 29.39, Time: 00:03:02 \n",
      "Ep. 300, Timestep 3525,  Ep.Timesteps 19, Score: 21.01, Avg.Score: 20.29, Max.Score: 36.75, Time: 00:03:10 \n",
      "Ep. 310, Timestep 3645,  Ep.Timesteps 9, Score: 18.18, Avg.Score: 20.09, Max.Score: 36.75, Time: 00:03:17 \n",
      "Ep. 320, Timestep 3771,  Ep.Timesteps 7, Score: 17.57, Avg.Score: 20.11, Max.Score: 36.75, Time: 00:03:22 \n",
      "Ep. 330, Timestep 3895,  Ep.Timesteps 9, Score: 20.13, Avg.Score: 20.11, Max.Score: 36.75, Time: 00:03:27 \n",
      "Ep. 340, Timestep 3995,  Ep.Timesteps 13, Score: 18.74, Avg.Score: 20.06, Max.Score: 36.75, Time: 00:03:32 \n",
      "Ep. 350, Timestep 4118,  Ep.Timesteps 8, Score: 20.96, Avg.Score: 20.21, Max.Score: 36.75, Time: 00:03:39 \n",
      "Ep. 360, Timestep 4246,  Ep.Timesteps 8, Score: 19.08, Avg.Score: 20.17, Max.Score: 36.75, Time: 00:03:46 \n",
      "Ep. 370, Timestep 4373,  Ep.Timesteps 18, Score: 24.89, Avg.Score: 20.12, Max.Score: 36.75, Time: 00:03:51 \n",
      "Ep. 380, Timestep 4480,  Ep.Timesteps 13, Score: 22.44, Avg.Score: 20.12, Max.Score: 36.75, Time: 00:03:56 \n",
      "Ep. 390, Timestep 4623,  Ep.Timesteps 16, Score: 25.11, Avg.Score: 20.14, Max.Score: 36.75, Time: 00:04:03 \n",
      "Ep. 400, Timestep 4740,  Ep.Timesteps 10, Score: 20.48, Avg.Score: 19.87, Max.Score: 28.59, Time: 00:04:06 \n",
      "Ep. 410, Timestep 4859,  Ep.Timesteps 7, Score: 18.34, Avg.Score: 20.07, Max.Score: 28.59, Time: 00:04:12 \n",
      "Ep. 420, Timestep 5010,  Ep.Timesteps 11, Score: 23.23, Avg.Score: 20.09, Max.Score: 28.59, Time: 00:04:19 \n",
      "Ep. 430, Timestep 5116,  Ep.Timesteps 14, Score: 16.06, Avg.Score: 20.01, Max.Score: 28.59, Time: 00:04:24 \n",
      "Ep. 440, Timestep 5218,  Ep.Timesteps 14, Score: 21.24, Avg.Score: 20.12, Max.Score: 28.59, Time: 00:04:28 \n",
      "Ep. 450, Timestep 5315,  Ep.Timesteps 13, Score: 22.84, Avg.Score: 20.09, Max.Score: 28.59, Time: 00:04:33 \n",
      "Ep. 460, Timestep 5429,  Ep.Timesteps 11, Score: 16.74, Avg.Score: 20.18, Max.Score: 28.59, Time: 00:04:40 \n",
      "Ep. 470, Timestep 5526,  Ep.Timesteps 12, Score: 20.53, Avg.Score: 20.00, Max.Score: 28.59, Time: 00:04:45 \n",
      "Ep. 480, Timestep 5628,  Ep.Timesteps 8, Score: 18.12, Avg.Score: 19.99, Max.Score: 28.59, Time: 00:04:50 \n",
      "Ep. 490, Timestep 5730,  Ep.Timesteps 7, Score: 17.62, Avg.Score: 19.89, Max.Score: 27.93, Time: 00:04:55 \n",
      "Ep. 500, Timestep 5860,  Ep.Timesteps 8, Score: 16.71, Avg.Score: 19.95, Max.Score: 27.72, Time: 00:05:04 \n",
      "Ep. 510, Timestep 6006,  Ep.Timesteps 9, Score: 19.21, Avg.Score: 19.96, Max.Score: 30.41, Time: 00:05:13 \n",
      "Ep. 520, Timestep 6130,  Ep.Timesteps 7, Score: 19.03, Avg.Score: 19.93, Max.Score: 30.41, Time: 00:05:20 \n",
      "Ep. 530, Timestep 6242,  Ep.Timesteps 8, Score: 19.10, Avg.Score: 19.99, Max.Score: 30.41, Time: 00:05:27 \n",
      "Ep. 540, Timestep 6375,  Ep.Timesteps 8, Score: 19.29, Avg.Score: 20.08, Max.Score: 41.22, Time: 00:05:33 \n",
      "Ep. 550, Timestep 6520,  Ep.Timesteps 15, Score: 17.76, Avg.Score: 20.29, Max.Score: 41.22, Time: 00:05:41 \n",
      "Ep. 560, Timestep 6631,  Ep.Timesteps 16, Score: 21.16, Avg.Score: 20.32, Max.Score: 41.22, Time: 00:05:47 \n",
      "Ep. 570, Timestep 6750,  Ep.Timesteps 6, Score: 19.19, Avg.Score: 20.50, Max.Score: 41.22, Time: 00:05:54 \n",
      "Ep. 580, Timestep 6872,  Ep.Timesteps 15, Score: 21.45, Avg.Score: 20.71, Max.Score: 41.22, Time: 00:06:01 \n",
      "Ep. 590, Timestep 6965,  Ep.Timesteps 9, Score: 18.52, Avg.Score: 20.63, Max.Score: 41.22, Time: 00:06:06 \n",
      "Ep. 600, Timestep 7109,  Ep.Timesteps 12, Score: 18.01, Avg.Score: 20.75, Max.Score: 41.22, Time: 00:06:14 \n",
      "Ep. 610, Timestep 7227,  Ep.Timesteps 11, Score: 20.97, Avg.Score: 20.57, Max.Score: 41.22, Time: 00:06:19 \n",
      "Ep. 620, Timestep 7359,  Ep.Timesteps 24, Score: 26.60, Avg.Score: 20.48, Max.Score: 41.22, Time: 00:06:26 \n",
      "Ep. 630, Timestep 7481,  Ep.Timesteps 12, Score: 21.72, Avg.Score: 20.47, Max.Score: 41.22, Time: 00:06:34 \n",
      "Ep. 640, Timestep 7600,  Ep.Timesteps 15, Score: 21.61, Avg.Score: 20.43, Max.Score: 31.83, Time: 00:06:41 \n",
      "Ep. 650, Timestep 7725,  Ep.Timesteps 8, Score: 18.24, Avg.Score: 20.22, Max.Score: 31.83, Time: 00:06:48 \n",
      "Ep. 660, Timestep 7866,  Ep.Timesteps 16, Score: 21.93, Avg.Score: 20.28, Max.Score: 31.83, Time: 00:06:56 \n",
      "Ep. 670, Timestep 7976,  Ep.Timesteps 17, Score: 24.64, Avg.Score: 20.22, Max.Score: 31.83, Time: 00:06:59 \n",
      "Ep. 680, Timestep 8076,  Ep.Timesteps 6, Score: 18.68, Avg.Score: 20.09, Max.Score: 31.83, Time: 00:07:05 \n",
      "Ep. 690, Timestep 8188,  Ep.Timesteps 9, Score: 20.24, Avg.Score: 20.09, Max.Score: 31.83, Time: 00:07:11 \n",
      "Ep. 700, Timestep 8274,  Ep.Timesteps 6, Score: 18.48, Avg.Score: 19.75, Max.Score: 31.83, Time: 00:07:16 \n",
      "Ep. 710, Timestep 8393,  Ep.Timesteps 12, Score: 24.20, Avg.Score: 19.80, Max.Score: 31.83, Time: 00:07:22 \n",
      "Ep. 720, Timestep 8506,  Ep.Timesteps 7, Score: 17.94, Avg.Score: 19.84, Max.Score: 31.83, Time: 00:07:30 \n",
      "Ep. 730, Timestep 8614,  Ep.Timesteps 14, Score: 18.13, Avg.Score: 19.76, Max.Score: 31.83, Time: 00:07:35 \n",
      "Ep. 740, Timestep 8751,  Ep.Timesteps 8, Score: 18.01, Avg.Score: 19.62, Max.Score: 27.61, Time: 00:07:41 \n",
      "Ep. 750, Timestep 8878,  Ep.Timesteps 11, Score: 20.11, Avg.Score: 19.81, Max.Score: 29.76, Time: 00:07:48 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 760, Timestep 8978,  Ep.Timesteps 18, Score: 23.16, Avg.Score: 19.58, Max.Score: 29.76, Time: 00:07:54 \n",
      "Ep. 770, Timestep 9104,  Ep.Timesteps 7, Score: 20.92, Avg.Score: 19.62, Max.Score: 29.76, Time: 00:08:02 \n",
      "Ep. 780, Timestep 9211,  Ep.Timesteps 8, Score: 21.54, Avg.Score: 19.64, Max.Score: 29.76, Time: 00:08:09 \n",
      "Ep. 790, Timestep 9361,  Ep.Timesteps 22, Score: 26.45, Avg.Score: 19.89, Max.Score: 29.76, Time: 00:08:17 \n",
      "Ep. 800, Timestep 9488,  Ep.Timesteps 8, Score: 21.27, Avg.Score: 20.04, Max.Score: 29.76, Time: 00:08:22 \n",
      "Ep. 810, Timestep 9627,  Ep.Timesteps 14, Score: 22.65, Avg.Score: 20.14, Max.Score: 37.90, Time: 00:08:28 \n",
      "Ep. 820, Timestep 9778,  Ep.Timesteps 8, Score: 20.16, Avg.Score: 20.14, Max.Score: 37.90, Time: 00:08:36 \n",
      "Ep. 830, Timestep 9906,  Ep.Timesteps 8, Score: 20.52, Avg.Score: 20.30, Max.Score: 37.90, Time: 00:08:44 \n",
      "Ep. 840, Timestep 10008,  Ep.Timesteps 4, Score: 12.58, Avg.Score: 20.19, Max.Score: 37.90, Time: 00:08:50 \n",
      "Ep. 850, Timestep 10048,  Ep.Timesteps 4, Score: 12.40, Avg.Score: 19.26, Max.Score: 37.90, Time: 00:08:53 \n",
      "Ep. 860, Timestep 10088,  Ep.Timesteps 4, Score: 12.22, Avg.Score: 18.63, Max.Score: 37.90, Time: 00:08:55 \n",
      "Ep. 870, Timestep 10128,  Ep.Timesteps 4, Score: 12.77, Avg.Score: 17.92, Max.Score: 37.90, Time: 00:08:58 \n",
      "Ep. 880, Timestep 10168,  Ep.Timesteps 4, Score: 12.03, Avg.Score: 17.22, Max.Score: 37.90, Time: 00:09:01 \n",
      "Ep. 890, Timestep 10208,  Ep.Timesteps 4, Score: 12.20, Avg.Score: 16.24, Max.Score: 37.90, Time: 00:09:03 \n",
      "Ep. 900, Timestep 10248,  Ep.Timesteps 4, Score: 12.84, Avg.Score: 15.54, Max.Score: 37.90, Time: 00:09:04 \n",
      "Ep. 910, Timestep 10288,  Ep.Timesteps 4, Score: 13.14, Avg.Score: 14.70, Max.Score: 26.52, Time: 00:09:05 \n",
      "Ep. 920, Timestep 10328,  Ep.Timesteps 4, Score: 12.25, Avg.Score: 13.90, Max.Score: 26.52, Time: 00:09:06 \n",
      "Ep. 930, Timestep 10368,  Ep.Timesteps 4, Score: 12.27, Avg.Score: 13.02, Max.Score: 26.52, Time: 00:09:08 \n",
      "Ep. 940, Timestep 10408,  Ep.Timesteps 4, Score: 12.66, Avg.Score: 12.40, Max.Score: 13.15, Time: 00:09:10 \n",
      "Ep. 950, Timestep 10448,  Ep.Timesteps 4, Score: 12.22, Avg.Score: 12.33, Max.Score: 13.15, Time: 00:09:13 \n",
      "Ep. 960, Timestep 10488,  Ep.Timesteps 4, Score: 12.67, Avg.Score: 12.33, Max.Score: 13.15, Time: 00:09:15 \n",
      "Ep. 970, Timestep 10528,  Ep.Timesteps 4, Score: 12.64, Avg.Score: 12.31, Max.Score: 13.15, Time: 00:09:17 \n",
      "Ep. 980, Timestep 10568,  Ep.Timesteps 4, Score: 11.93, Avg.Score: 12.30, Max.Score: 13.15, Time: 00:09:20 \n",
      "Ep. 990, Timestep 10608,  Ep.Timesteps 4, Score: 12.54, Avg.Score: 12.30, Max.Score: 13.15, Time: 00:09:22 \n",
      "Ep. 1000, Timestep 10648,  Ep.Timesteps 4, Score: 12.93, Avg.Score: 12.32, Max.Score: 13.14, Time: 00:09:25 \n",
      "Ep. 1010, Timestep 10688,  Ep.Timesteps 4, Score: 12.75, Avg.Score: 12.28, Max.Score: 12.97, Time: 00:09:28 \n",
      "Ep. 1020, Timestep 10728,  Ep.Timesteps 4, Score: 12.30, Avg.Score: 12.31, Max.Score: 13.07, Time: 00:09:31 \n",
      "Ep. 1030, Timestep 10768,  Ep.Timesteps 4, Score: 12.05, Avg.Score: 12.33, Max.Score: 13.07, Time: 00:09:35 \n",
      "Ep. 1040, Timestep 10808,  Ep.Timesteps 4, Score: 12.89, Avg.Score: 12.33, Max.Score: 13.07, Time: 00:09:38 \n",
      "Ep. 1050, Timestep 10848,  Ep.Timesteps 4, Score: 12.21, Avg.Score: 12.43, Max.Score: 13.07, Time: 00:09:40 \n",
      "Ep. 1060, Timestep 10888,  Ep.Timesteps 4, Score: 12.60, Avg.Score: 12.42, Max.Score: 13.07, Time: 00:09:43 \n",
      "Ep. 1070, Timestep 10928,  Ep.Timesteps 4, Score: 12.09, Avg.Score: 12.41, Max.Score: 13.07, Time: 00:09:46 \n",
      "Ep. 1080, Timestep 10968,  Ep.Timesteps 4, Score: 12.06, Avg.Score: 12.43, Max.Score: 13.07, Time: 00:09:48 \n",
      "Ep. 1090, Timestep 11008,  Ep.Timesteps 4, Score: 12.57, Avg.Score: 12.42, Max.Score: 13.07, Time: 00:09:50 \n",
      "Ep. 1100, Timestep 11048,  Ep.Timesteps 4, Score: 12.91, Avg.Score: 12.42, Max.Score: 13.07, Time: 00:09:51 \n",
      "Ep. 1110, Timestep 11088,  Ep.Timesteps 4, Score: 12.11, Avg.Score: 12.44, Max.Score: 13.07, Time: 00:09:52 \n",
      "Ep. 1120, Timestep 11128,  Ep.Timesteps 4, Score: 11.23, Avg.Score: 12.42, Max.Score: 13.04, Time: 00:09:55 \n",
      "Ep. 1130, Timestep 11168,  Ep.Timesteps 4, Score: 12.30, Avg.Score: 12.42, Max.Score: 13.04, Time: 00:09:57 \n",
      "Ep. 1140, Timestep 11208,  Ep.Timesteps 4, Score: 12.28, Avg.Score: 12.41, Max.Score: 13.04, Time: 00:09:59 \n",
      "Ep. 1150, Timestep 11248,  Ep.Timesteps 4, Score: 11.91, Avg.Score: 12.39, Max.Score: 13.02, Time: 00:10:01 \n",
      "Ep. 1160, Timestep 11288,  Ep.Timesteps 4, Score: 11.72, Avg.Score: 12.34, Max.Score: 13.02, Time: 00:10:04 \n",
      "Ep. 1170, Timestep 11328,  Ep.Timesteps 4, Score: 12.81, Avg.Score: 12.35, Max.Score: 13.02, Time: 00:10:06 \n",
      "Ep. 1180, Timestep 11368,  Ep.Timesteps 4, Score: 12.68, Avg.Score: 12.35, Max.Score: 13.02, Time: 00:10:08 \n",
      "Ep. 1190, Timestep 11408,  Ep.Timesteps 4, Score: 10.49, Avg.Score: 12.33, Max.Score: 13.02, Time: 00:10:10 \n",
      "Ep. 1200, Timestep 11448,  Ep.Timesteps 4, Score: 12.83, Avg.Score: 12.32, Max.Score: 13.02, Time: 00:10:12 \n",
      "Ep. 1210, Timestep 11488,  Ep.Timesteps 4, Score: 12.11, Avg.Score: 12.30, Max.Score: 13.02, Time: 00:10:14 \n",
      "Ep. 1220, Timestep 11528,  Ep.Timesteps 4, Score: 12.69, Avg.Score: 12.31, Max.Score: 13.02, Time: 00:10:17 \n",
      "Ep. 1230, Timestep 11568,  Ep.Timesteps 4, Score: 12.17, Avg.Score: 12.32, Max.Score: 13.02, Time: 00:10:19 \n",
      "Ep. 1240, Timestep 11608,  Ep.Timesteps 4, Score: 12.67, Avg.Score: 12.33, Max.Score: 12.98, Time: 00:10:21 \n",
      "Ep. 1250, Timestep 11648,  Ep.Timesteps 4, Score: 12.80, Avg.Score: 12.33, Max.Score: 12.98, Time: 00:10:23 \n",
      "Ep. 1260, Timestep 11688,  Ep.Timesteps 4, Score: 9.25, Avg.Score: 12.35, Max.Score: 13.02, Time: 00:10:26 \n",
      "Ep. 1270, Timestep 11728,  Ep.Timesteps 4, Score: 12.26, Avg.Score: 12.38, Max.Score: 13.02, Time: 00:10:28 \n",
      "Ep. 1280, Timestep 11768,  Ep.Timesteps 4, Score: 12.91, Avg.Score: 12.39, Max.Score: 13.13, Time: 00:10:31 \n",
      "Ep. 1290, Timestep 11808,  Ep.Timesteps 4, Score: 12.43, Avg.Score: 12.42, Max.Score: 13.13, Time: 00:10:33 \n",
      "Ep. 1300, Timestep 11848,  Ep.Timesteps 4, Score: 12.63, Avg.Score: 12.42, Max.Score: 13.13, Time: 00:10:34 \n",
      "Ep. 1310, Timestep 11888,  Ep.Timesteps 4, Score: 12.90, Avg.Score: 12.43, Max.Score: 13.13, Time: 00:10:35 \n",
      "Ep. 1320, Timestep 11928,  Ep.Timesteps 4, Score: 12.72, Avg.Score: 12.43, Max.Score: 13.13, Time: 00:10:36 \n",
      "Ep. 1330, Timestep 11968,  Ep.Timesteps 4, Score: 12.86, Avg.Score: 12.43, Max.Score: 13.13, Time: 00:10:38 \n",
      "Ep. 1340, Timestep 12008,  Ep.Timesteps 4, Score: 12.19, Avg.Score: 12.41, Max.Score: 13.13, Time: 00:10:40 \n",
      "Ep. 1350, Timestep 12048,  Ep.Timesteps 4, Score: 12.13, Avg.Score: 12.42, Max.Score: 13.13, Time: 00:10:42 \n",
      "Ep. 1360, Timestep 12088,  Ep.Timesteps 4, Score: 12.93, Avg.Score: 12.46, Max.Score: 13.13, Time: 00:10:44 \n",
      "Ep. 1370, Timestep 12128,  Ep.Timesteps 4, Score: 12.86, Avg.Score: 12.46, Max.Score: 13.13, Time: 00:10:46 \n",
      "Ep. 1380, Timestep 12168,  Ep.Timesteps 4, Score: 12.83, Avg.Score: 12.45, Max.Score: 13.09, Time: 00:10:48 \n",
      "Ep. 1390, Timestep 12208,  Ep.Timesteps 4, Score: 12.54, Avg.Score: 12.44, Max.Score: 13.09, Time: 00:10:50 \n",
      "Ep. 1400, Timestep 12248,  Ep.Timesteps 4, Score: 11.58, Avg.Score: 12.44, Max.Score: 13.09, Time: 00:10:52 \n",
      "Ep. 1410, Timestep 12288,  Ep.Timesteps 4, Score: 11.87, Avg.Score: 12.44, Max.Score: 13.09, Time: 00:10:54 \n",
      "Ep. 1420, Timestep 12328,  Ep.Timesteps 4, Score: 12.87, Avg.Score: 12.42, Max.Score: 13.09, Time: 00:10:56 \n",
      "Ep. 1430, Timestep 12368,  Ep.Timesteps 4, Score: 12.80, Avg.Score: 12.44, Max.Score: 13.10, Time: 00:10:59 \n",
      "Ep. 1440, Timestep 12408,  Ep.Timesteps 4, Score: 12.79, Avg.Score: 12.45, Max.Score: 13.10, Time: 00:11:01 \n",
      "Ep. 1450, Timestep 12448,  Ep.Timesteps 4, Score: 12.89, Avg.Score: 12.43, Max.Score: 13.10, Time: 00:11:03 \n",
      "Ep. 1460, Timestep 12488,  Ep.Timesteps 4, Score: 12.02, Avg.Score: 12.44, Max.Score: 13.10, Time: 00:11:05 \n",
      "Ep. 1470, Timestep 12528,  Ep.Timesteps 4, Score: 12.28, Avg.Score: 12.37, Max.Score: 13.10, Time: 00:11:07 \n",
      "Ep. 1480, Timestep 12568,  Ep.Timesteps 4, Score: 12.58, Avg.Score: 12.38, Max.Score: 13.10, Time: 00:11:09 \n",
      "Ep. 1490, Timestep 12608,  Ep.Timesteps 4, Score: 13.02, Avg.Score: 12.39, Max.Score: 13.10, Time: 00:11:11 \n",
      "Ep. 1500, Timestep 12648,  Ep.Timesteps 4, Score: 12.68, Avg.Score: 12.39, Max.Score: 13.10, Time: 00:11:13 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 1510, Timestep 12688,  Ep.Timesteps 4, Score: 12.54, Avg.Score: 12.35, Max.Score: 13.10, Time: 00:11:15 \n",
      "Ep. 1520, Timestep 12728,  Ep.Timesteps 4, Score: 12.18, Avg.Score: 12.35, Max.Score: 13.10, Time: 00:11:17 \n",
      "Ep. 1530, Timestep 12768,  Ep.Timesteps 4, Score: 12.05, Avg.Score: 12.32, Max.Score: 13.07, Time: 00:11:19 \n",
      "Ep. 1540, Timestep 12808,  Ep.Timesteps 4, Score: 12.87, Avg.Score: 12.32, Max.Score: 13.07, Time: 00:11:21 \n",
      "Ep. 1550, Timestep 12848,  Ep.Timesteps 4, Score: 12.04, Avg.Score: 12.28, Max.Score: 13.07, Time: 00:11:23 \n",
      "Ep. 1560, Timestep 12888,  Ep.Timesteps 4, Score: 12.81, Avg.Score: 12.29, Max.Score: 13.07, Time: 00:11:24 \n",
      "Ep. 1570, Timestep 12928,  Ep.Timesteps 4, Score: 12.83, Avg.Score: 12.36, Max.Score: 13.07, Time: 00:11:25 \n",
      "Ep. 1580, Timestep 12968,  Ep.Timesteps 4, Score: 12.66, Avg.Score: 12.37, Max.Score: 13.07, Time: 00:11:26 \n",
      "Ep. 1590, Timestep 13008,  Ep.Timesteps 4, Score: 12.80, Avg.Score: 12.36, Max.Score: 13.07, Time: 00:11:28 \n",
      "Ep. 1600, Timestep 13048,  Ep.Timesteps 4, Score: 13.19, Avg.Score: 12.37, Max.Score: 13.19, Time: 00:11:30 \n",
      "Ep. 1610, Timestep 13088,  Ep.Timesteps 4, Score: 12.86, Avg.Score: 12.40, Max.Score: 13.19, Time: 00:11:33 \n",
      "Ep. 1620, Timestep 13128,  Ep.Timesteps 4, Score: 12.85, Avg.Score: 12.43, Max.Score: 13.19, Time: 00:11:35 \n",
      "Ep. 1630, Timestep 13168,  Ep.Timesteps 4, Score: 12.57, Avg.Score: 12.43, Max.Score: 13.19, Time: 00:11:38 \n",
      "Ep. 1640, Timestep 13208,  Ep.Timesteps 4, Score: 12.16, Avg.Score: 12.44, Max.Score: 13.19, Time: 00:11:40 \n",
      "Ep. 1650, Timestep 13248,  Ep.Timesteps 4, Score: 12.28, Avg.Score: 12.49, Max.Score: 13.19, Time: 00:11:43 \n",
      "Ep. 1660, Timestep 13288,  Ep.Timesteps 4, Score: 12.81, Avg.Score: 12.46, Max.Score: 13.19, Time: 00:11:45 \n",
      "Ep. 1670, Timestep 13328,  Ep.Timesteps 4, Score: 11.93, Avg.Score: 12.44, Max.Score: 13.19, Time: 00:11:48 \n",
      "Ep. 1680, Timestep 13368,  Ep.Timesteps 4, Score: 12.74, Avg.Score: 12.43, Max.Score: 13.19, Time: 00:11:50 \n",
      "Ep. 1690, Timestep 13408,  Ep.Timesteps 4, Score: 12.37, Avg.Score: 12.43, Max.Score: 13.19, Time: 00:11:53 \n",
      "Ep. 1700, Timestep 13448,  Ep.Timesteps 4, Score: 12.11, Avg.Score: 12.42, Max.Score: 13.01, Time: 00:11:55 \n",
      "Ep. 1710, Timestep 13488,  Ep.Timesteps 4, Score: 12.56, Avg.Score: 12.42, Max.Score: 13.01, Time: 00:11:58 \n",
      "Ep. 1720, Timestep 13528,  Ep.Timesteps 4, Score: 12.69, Avg.Score: 12.41, Max.Score: 13.01, Time: 00:12:01 \n",
      "Ep. 1730, Timestep 13568,  Ep.Timesteps 4, Score: 13.14, Avg.Score: 12.43, Max.Score: 13.14, Time: 00:12:03 \n",
      "Ep. 1740, Timestep 13608,  Ep.Timesteps 4, Score: 12.25, Avg.Score: 12.41, Max.Score: 13.14, Time: 00:12:05 \n",
      "Ep. 1750, Timestep 13648,  Ep.Timesteps 4, Score: 12.28, Avg.Score: 12.38, Max.Score: 13.14, Time: 00:12:06 \n",
      "Ep. 1760, Timestep 13688,  Ep.Timesteps 4, Score: 12.54, Avg.Score: 12.39, Max.Score: 13.14, Time: 00:12:08 \n",
      "Ep. 1770, Timestep 13728,  Ep.Timesteps 4, Score: 12.83, Avg.Score: 12.35, Max.Score: 13.14, Time: 00:12:10 \n",
      "Ep. 1780, Timestep 13768,  Ep.Timesteps 4, Score: 12.70, Avg.Score: 12.34, Max.Score: 13.14, Time: 00:12:13 \n",
      "Ep. 1790, Timestep 13808,  Ep.Timesteps 4, Score: 12.12, Avg.Score: 12.34, Max.Score: 13.14, Time: 00:12:15 \n",
      "Ep. 1800, Timestep 13848,  Ep.Timesteps 4, Score: 12.91, Avg.Score: 12.36, Max.Score: 13.14, Time: 00:12:17 \n",
      "Ep. 1810, Timestep 13888,  Ep.Timesteps 4, Score: 12.24, Avg.Score: 12.37, Max.Score: 13.14, Time: 00:12:20 \n",
      "Ep. 1820, Timestep 13928,  Ep.Timesteps 4, Score: 12.71, Avg.Score: 12.34, Max.Score: 13.14, Time: 00:12:22 \n",
      "Ep. 1830, Timestep 13968,  Ep.Timesteps 4, Score: 12.76, Avg.Score: 12.34, Max.Score: 13.00, Time: 00:12:24 \n",
      "Ep. 1840, Timestep 14008,  Ep.Timesteps 4, Score: 12.49, Avg.Score: 12.36, Max.Score: 13.00, Time: 00:12:26 \n",
      "Ep. 1850, Timestep 14048,  Ep.Timesteps 4, Score: 12.74, Avg.Score: 12.37, Max.Score: 13.00, Time: 00:12:29 \n",
      "Ep. 1860, Timestep 14088,  Ep.Timesteps 4, Score: 12.80, Avg.Score: 12.39, Max.Score: 13.00, Time: 00:12:32 \n",
      "Ep. 1870, Timestep 14128,  Ep.Timesteps 4, Score: 12.24, Avg.Score: 12.45, Max.Score: 13.09, Time: 00:12:34 \n",
      "Ep. 1880, Timestep 14168,  Ep.Timesteps 4, Score: 12.66, Avg.Score: 12.45, Max.Score: 13.09, Time: 00:12:37 \n",
      "Ep. 1890, Timestep 14208,  Ep.Timesteps 4, Score: 10.45, Avg.Score: 12.47, Max.Score: 13.09, Time: 00:12:39 \n",
      "Ep. 1900, Timestep 14248,  Ep.Timesteps 4, Score: 11.34, Avg.Score: 12.42, Max.Score: 13.11, Time: 00:12:41 \n",
      "Ep. 1910, Timestep 14288,  Ep.Timesteps 4, Score: 12.77, Avg.Score: 12.42, Max.Score: 13.11, Time: 00:12:44 \n",
      "Ep. 1920, Timestep 14328,  Ep.Timesteps 4, Score: 11.98, Avg.Score: 12.46, Max.Score: 13.11, Time: 00:12:46 \n",
      "Ep. 1930, Timestep 14368,  Ep.Timesteps 4, Score: 11.00, Avg.Score: 12.44, Max.Score: 13.11, Time: 00:12:47 \n",
      "Ep. 1940, Timestep 14412,  Ep.Timesteps 5, Score: 16.00, Avg.Score: 12.56, Max.Score: 16.00, Time: 00:12:49 \n",
      "Ep. 1950, Timestep 14462,  Ep.Timesteps 5, Score: 16.19, Avg.Score: 12.87, Max.Score: 16.19, Time: 00:12:52 \n",
      "Ep. 1960, Timestep 14512,  Ep.Timesteps 5, Score: 16.52, Avg.Score: 13.20, Max.Score: 16.59, Time: 00:12:55 \n",
      "Ep. 1970, Timestep 14575,  Ep.Timesteps 8, Score: 22.25, Avg.Score: 13.94, Max.Score: 23.08, Time: 00:12:59 \n",
      "Ep. 1980, Timestep 14657,  Ep.Timesteps 9, Score: 25.25, Avg.Score: 15.05, Max.Score: 25.25, Time: 00:13:04 \n",
      "Ep. 1990, Timestep 14738,  Ep.Timesteps 9, Score: 25.35, Avg.Score: 16.15, Max.Score: 25.35, Time: 00:13:08 \n",
      "Ep. 2000, Timestep 14816,  Ep.Timesteps 8, Score: 23.90, Avg.Score: 17.18, Max.Score: 25.35, Time: 00:13:13 \n",
      "Ep. 2010, Timestep 14897,  Ep.Timesteps 8, Score: 24.12, Avg.Score: 18.29, Max.Score: 26.14, Time: 00:13:17 \n",
      "Ep. 2020, Timestep 14973,  Ep.Timesteps 7, Score: 22.63, Avg.Score: 19.33, Max.Score: 26.14, Time: 00:13:22 \n",
      "Ep. 2030, Timestep 15046,  Ep.Timesteps 8, Score: 24.99, Avg.Score: 20.41, Max.Score: 26.14, Time: 00:13:26 \n",
      "Ep. 2040, Timestep 15117,  Ep.Timesteps 7, Score: 22.66, Avg.Score: 21.36, Max.Score: 26.14, Time: 00:13:29 \n",
      "Ep. 2050, Timestep 15186,  Ep.Timesteps 7, Score: 23.28, Avg.Score: 22.05, Max.Score: 26.14, Time: 00:13:33 \n",
      "Ep. 2060, Timestep 15251,  Ep.Timesteps 6, Score: 19.79, Avg.Score: 22.62, Max.Score: 26.14, Time: 00:13:37 \n",
      "Ep. 2070, Timestep 15314,  Ep.Timesteps 6, Score: 21.37, Avg.Score: 22.73, Max.Score: 26.14, Time: 00:13:40 \n",
      "Ep. 2080, Timestep 15379,  Ep.Timesteps 6, Score: 20.51, Avg.Score: 22.54, Max.Score: 26.14, Time: 00:13:43 \n",
      "Ep. 2090, Timestep 15445,  Ep.Timesteps 6, Score: 21.06, Avg.Score: 22.35, Max.Score: 26.14, Time: 00:13:47 \n",
      "Ep. 2100, Timestep 15511,  Ep.Timesteps 7, Score: 22.85, Avg.Score: 22.27, Max.Score: 26.14, Time: 00:13:50 \n",
      "Ep. 2110, Timestep 15579,  Ep.Timesteps 7, Score: 23.03, Avg.Score: 22.13, Max.Score: 25.85, Time: 00:13:53 \n",
      "Ep. 2120, Timestep 15648,  Ep.Timesteps 6, Score: 20.37, Avg.Score: 22.07, Max.Score: 25.18, Time: 00:13:57 \n",
      "Ep. 2130, Timestep 15717,  Ep.Timesteps 7, Score: 22.78, Avg.Score: 21.99, Max.Score: 25.18, Time: 00:14:01 \n",
      "Ep. 2140, Timestep 15790,  Ep.Timesteps 8, Score: 23.81, Avg.Score: 21.97, Max.Score: 23.81, Time: 00:14:05 \n",
      "Ep. 2150, Timestep 15865,  Ep.Timesteps 8, Score: 23.36, Avg.Score: 21.97, Max.Score: 23.89, Time: 00:14:08 \n",
      "Ep. 2160, Timestep 15943,  Ep.Timesteps 8, Score: 22.46, Avg.Score: 22.07, Max.Score: 23.89, Time: 00:14:12 \n",
      "Ep. 2170, Timestep 16050,  Ep.Timesteps 29, Score: 47.02, Avg.Score: 22.58, Max.Score: 47.02, Time: 00:14:15 \n",
      "Ep. 2180, Timestep 16265,  Ep.Timesteps 16, Score: 32.68, Avg.Score: 24.22, Max.Score: 50.83, Time: 00:14:27 \n",
      "Ep. 2190, Timestep 16663,  Ep.Timesteps 54, Score: 76.57, Avg.Score: 28.40, Max.Score: 82.26, Time: 00:14:48 \n",
      "Ep. 2200, Timestep 17165,  Ep.Timesteps 69, Score: 92.43, Avg.Score: 33.66, Max.Score: 92.43, Time: 00:15:11 \n",
      "Ep. 2210, Timestep 17577,  Ep.Timesteps 45, Score: 74.03, Avg.Score: 38.09, Max.Score: 92.43, Time: 00:15:36 \n",
      "Ep. 2220, Timestep 18044,  Ep.Timesteps 57, Score: 87.43, Avg.Score: 43.25, Max.Score: 92.43, Time: 00:16:19 \n",
      "Ep. 2230, Timestep 18615,  Ep.Timesteps 62, Score: 100.49, Avg.Score: 49.92, Max.Score: 125.18, Time: 00:17:04 \n",
      "Ep. 2240, Timestep 19064,  Ep.Timesteps 52, Score: 81.27, Avg.Score: 55.11, Max.Score: 125.18, Time: 00:17:34 \n",
      "Ep. 2250, Timestep 19577,  Ep.Timesteps 59, Score: 89.02, Avg.Score: 61.41, Max.Score: 125.18, Time: 00:17:58 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 2260, Timestep 20104,  Ep.Timesteps 55, Score: 100.49, Avg.Score: 68.30, Max.Score: 125.18, Time: 00:18:25 \n",
      "Ep. 2270, Timestep 20647,  Ep.Timesteps 62, Score: 89.79, Avg.Score: 74.62, Max.Score: 125.18, Time: 00:18:54 \n",
      "Ep. 2280, Timestep 21172,  Ep.Timesteps 55, Score: 75.90, Avg.Score: 78.20, Max.Score: 125.18, Time: 00:19:23 \n",
      "Ep. 2290, Timestep 21685,  Ep.Timesteps 54, Score: 76.15, Avg.Score: 78.65, Max.Score: 125.18, Time: 00:19:51 \n",
      "Ep. 2300, Timestep 22195,  Ep.Timesteps 48, Score: 60.08, Avg.Score: 78.23, Max.Score: 125.18, Time: 00:20:21 \n",
      "Ep. 2310, Timestep 22706,  Ep.Timesteps 48, Score: 65.09, Avg.Score: 78.84, Max.Score: 125.18, Time: 00:20:48 \n",
      "Ep. 2320, Timestep 23200,  Ep.Timesteps 55, Score: 92.72, Avg.Score: 79.60, Max.Score: 125.18, Time: 00:21:15 \n",
      "Ep. 2330, Timestep 23711,  Ep.Timesteps 44, Score: 79.88, Avg.Score: 79.65, Max.Score: 105.30, Time: 00:21:45 \n",
      "Ep. 2340, Timestep 24230,  Ep.Timesteps 51, Score: 98.43, Avg.Score: 81.65, Max.Score: 116.22, Time: 00:22:10 \n",
      "Ep. 2350, Timestep 24806,  Ep.Timesteps 59, Score: 110.77, Avg.Score: 83.56, Max.Score: 123.70, Time: 00:22:36 \n",
      "Ep. 2360, Timestep 25516,  Ep.Timesteps 74, Score: 118.79, Avg.Score: 86.71, Max.Score: 138.87, Time: 00:23:16 \n",
      "Ep. 2370, Timestep 26397,  Ep.Timesteps 86, Score: 142.42, Avg.Score: 91.50, Max.Score: 150.18, Time: 00:24:01 \n",
      "Ep. 2380, Timestep 28237,  Ep.Timesteps 1000, Score: 1010.91, Avg.Score: 107.29, Max.Score: 1010.91, Time: 00:24:55 \n",
      "Ep. 2390, Timestep 29255,  Ep.Timesteps 67, Score: 114.18, Avg.Score: 115.18, Max.Score: 1010.91, Time: 00:26:30 \n",
      "Ep. 2400, Timestep 32925,  Ep.Timesteps 102, Score: 142.00, Avg.Score: 145.74, Max.Score: 1010.91, Time: 00:29:01 \n",
      "Ep. 2410, Timestep 33763,  Ep.Timesteps 78, Score: 122.86, Avg.Score: 151.61, Max.Score: 1010.91, Time: 00:29:34 \n",
      "Ep. 2420, Timestep 34715,  Ep.Timesteps 123, Score: 170.41, Avg.Score: 157.93, Max.Score: 1010.91, Time: 00:30:11 \n",
      "Ep. 2430, Timestep 35686,  Ep.Timesteps 101, Score: 156.01, Avg.Score: 164.20, Max.Score: 1010.91, Time: 00:30:46 \n",
      "Ep. 2440, Timestep 36686,  Ep.Timesteps 82, Score: 143.22, Avg.Score: 170.49, Max.Score: 1010.91, Time: 00:31:36 \n",
      "Ep. 2450, Timestep 37578,  Ep.Timesteps 75, Score: 117.84, Avg.Score: 175.13, Max.Score: 1010.91, Time: 00:32:24 \n",
      "Ep. 2460, Timestep 38701,  Ep.Timesteps 78, Score: 136.81, Avg.Score: 180.99, Max.Score: 1010.91, Time: 00:33:26 \n",
      "Ep. 2470, Timestep 39885,  Ep.Timesteps 128, Score: 193.07, Avg.Score: 185.41, Max.Score: 1010.91, Time: 00:34:25 \n",
      "Ep. 2480, Timestep 41085,  Ep.Timesteps 209, Score: 274.20, Avg.Score: 180.52, Max.Score: 916.80, Time: 00:35:26 \n",
      "Ep. 2490, Timestep 43349,  Ep.Timesteps 122, Score: 174.09, Avg.Score: 193.03, Max.Score: 1047.28, Time: 00:37:24 \n",
      "Ep. 2500, Timestep 44846,  Ep.Timesteps 214, Score: 340.71, Avg.Score: 176.18, Max.Score: 1047.28, Time: 00:38:44 \n",
      "Ep. 2510, Timestep 51002,  Ep.Timesteps 245, Score: 365.95, Avg.Score: 247.11, Max.Score: 1715.34, Time: 00:44:09 \n",
      "Ep. 2520, Timestep 55925,  Ep.Timesteps 1000, Score: 1290.24, Avg.Score: 300.31, Max.Score: 1715.34, Time: 00:47:58 \n",
      "Ep. 2530, Timestep 64341,  Ep.Timesteps 1000, Score: 1539.20, Avg.Score: 397.66, Max.Score: 1715.34, Time: 00:54:33 \n",
      "Ep. 2540, Timestep 71843,  Ep.Timesteps 1000, Score: 1519.27, Avg.Score: 509.12, Max.Score: 1982.20, Time: 00:59:27 \n",
      "Ep. 2550, Timestep 79197,  Ep.Timesteps 217, Score: 414.06, Avg.Score: 619.32, Max.Score: 1982.20, Time: 01:05:12 \n",
      "Ep. 2560, Timestep 88466,  Ep.Timesteps 1000, Score: 1943.30, Avg.Score: 755.05, Max.Score: 1982.20, Time: 01:11:21 \n",
      "Ep. 2570, Timestep 96322,  Ep.Timesteps 1000, Score: 1823.97, Avg.Score: 876.07, Max.Score: 2098.89, Time: 01:17:15 \n",
      "Ep. 2580, Timestep 103909,  Ep.Timesteps 1000, Score: 1731.99, Avg.Score: 1000.78, Max.Score: 2098.89, Time: 01:22:52 \n",
      "Ep. 2590, Timestep 113909,  Ep.Timesteps 1000, Score: 2092.28, Avg.Score: 1172.65, Max.Score: 2103.73, Time: 01:31:47 \n",
      "Ep. 2600, Timestep 123909,  Ep.Timesteps 1000, Score: 1931.28, Avg.Score: 1346.64, Max.Score: 2103.73, Time: 01:41:23 \n",
      "Ep. 2610, Timestep 131878,  Ep.Timesteps 1000, Score: 2194.54, Avg.Score: 1435.55, Max.Score: 2334.01, Time: 01:48:24 \n",
      "Ep. 2620, Timestep 141695,  Ep.Timesteps 850, Score: 1983.82, Avg.Score: 1591.02, Max.Score: 2471.16, Time: 01:58:29 \n",
      "Ep. 2630, Timestep 148542,  Ep.Timesteps 255, Score: 597.12, Avg.Score: 1633.01, Max.Score: 2471.16, Time: 02:06:06 \n",
      "Ep. 2640, Timestep 156567,  Ep.Timesteps 1000, Score: 2279.11, Avg.Score: 1690.41, Max.Score: 2471.16, Time: 02:13:48 \n",
      "Ep. 2650, Timestep 166540,  Ep.Timesteps 1000, Score: 2395.97, Avg.Score: 1792.10, Max.Score: 2471.16, Time: 02:24:13 \n",
      "Ep. 2660, Timestep 175834,  Ep.Timesteps 1000, Score: 2279.73, Avg.Score: 1851.94, Max.Score: 2471.16, Time: 02:34:37 \n",
      "Ep. 2670, Timestep 185834,  Ep.Timesteps 1000, Score: 2364.94, Avg.Score: 1942.65, Max.Score: 2471.16, Time: 02:44:50 \n",
      "Ep. 2680, Timestep 195257,  Ep.Timesteps 1000, Score: 2418.86, Avg.Score: 2007.83, Max.Score: 2471.16, Time: 02:54:19 \n",
      "Ep. 2690, Timestep 203962,  Ep.Timesteps 1000, Score: 2328.86, Avg.Score: 2002.86, Max.Score: 2471.16, Time: 03:03:32 \n",
      "Ep. 2700, Timestep 213322,  Ep.Timesteps 360, Score: 823.43, Avg.Score: 2034.56, Max.Score: 2480.60, Time: 03:16:14 \n",
      "Ep. 2710, Timestep 221590,  Ep.Timesteps 1000, Score: 2487.38, Avg.Score: 2060.22, Max.Score: 2487.38, Time: 03:24:53 \n",
      "Ep. 2720, Timestep 228418,  Ep.Timesteps 616, Score: 1516.39, Avg.Score: 2004.61, Max.Score: 2511.98, Time: 03:33:41 \n",
      "Ep. 2730, Timestep 237155,  Ep.Timesteps 1000, Score: 2369.72, Avg.Score: 2063.74, Max.Score: 2536.05, Time: 03:41:57 \n",
      "Ep. 2740, Timestep 245278,  Ep.Timesteps 1000, Score: 2337.31, Avg.Score: 2066.74, Max.Score: 2536.05, Time: 03:49:48 \n",
      "Ep. 2750, Timestep 252734,  Ep.Timesteps 84, Score: 117.55, Avg.Score: 2000.28, Max.Score: 2536.05, Time: 03:59:35 \n",
      "Ep. 2760, Timestep 262734,  Ep.Timesteps 1000, Score: 2124.18, Avg.Score: 2007.19, Max.Score: 2536.05, Time: 04:09:46 \n",
      "Ep. 2770, Timestep 272521,  Ep.Timesteps 1000, Score: 2253.49, Avg.Score: 2002.02, Max.Score: 2536.05, Time: 04:21:38 \n",
      "Ep. 2780, Timestep 282521,  Ep.Timesteps 1000, Score: 2534.40, Avg.Score: 2034.93, Max.Score: 2539.00, Time: 04:33:14 \n",
      "Ep. 2790, Timestep 290792,  Ep.Timesteps 1000, Score: 2497.36, Avg.Score: 2044.37, Max.Score: 2554.78, Time: 04:42:49 \n",
      "Ep. 2800, Timestep 300009,  Ep.Timesteps 950, Score: 2213.17, Avg.Score: 2036.27, Max.Score: 2554.78, Time: 04:53:25 \n",
      "Ep. 2810, Timestep 309617,  Ep.Timesteps 1000, Score: 2408.02, Avg.Score: 2069.35, Max.Score: 2554.78, Time: 05:04:02 \n",
      "Ep. 2820, Timestep 318438,  Ep.Timesteps 959, Score: 2435.45, Avg.Score: 2117.05, Max.Score: 2554.78, Time: 05:14:36 \n",
      "Ep. 2830, Timestep 328438,  Ep.Timesteps 1000, Score: 2405.34, Avg.Score: 2151.13, Max.Score: 2560.09, Time: 05:26:18 \n",
      "Ep. 2840, Timestep 338438,  Ep.Timesteps 1000, Score: 2424.29, Avg.Score: 2205.04, Max.Score: 2560.09, Time: 05:36:59 \n",
      "Ep. 2850, Timestep 347958,  Ep.Timesteps 1000, Score: 2451.60, Avg.Score: 2269.72, Max.Score: 2560.09, Time: 05:45:55 \n",
      "Ep. 2860, Timestep 357539,  Ep.Timesteps 1000, Score: 2465.90, Avg.Score: 2275.72, Max.Score: 2560.09, Time: 05:53:47 \n",
      "Ep. 2870, Timestep 367238,  Ep.Timesteps 1000, Score: 2567.41, Avg.Score: 2284.29, Max.Score: 2567.41, Time: 06:01:13 \n",
      "Ep. 2880, Timestep 377238,  Ep.Timesteps 1000, Score: 2374.55, Avg.Score: 2282.68, Max.Score: 2567.41, Time: 06:09:00 \n",
      "Ep. 2890, Timestep 385943,  Ep.Timesteps 1000, Score: 2309.88, Avg.Score: 2288.99, Max.Score: 2567.41, Time: 06:15:39 \n",
      "Ep. 2900, Timestep 395943,  Ep.Timesteps 1000, Score: 2402.08, Avg.Score: 2312.77, Max.Score: 2567.41, Time: 06:23:21 \n",
      "Ep. 2910, Timestep 405632,  Ep.Timesteps 1000, Score: 2452.92, Avg.Score: 2312.94, Max.Score: 2567.41, Time: 06:30:35 \n",
      "Ep. 2920, Timestep 415134,  Ep.Timesteps 1000, Score: 2431.06, Avg.Score: 2329.20, Max.Score: 2567.41, Time: 06:43:19 \n",
      "Ep. 2930, Timestep 424475,  Ep.Timesteps 1000, Score: 2516.12, Avg.Score: 2311.20, Max.Score: 2567.41, Time: 06:51:01 \n",
      "Ep. 2940, Timestep 433672,  Ep.Timesteps 197, Score: 446.55, Avg.Score: 2294.86, Max.Score: 2567.41, Time: 06:56:40 \n",
      "Ep. 2950, Timestep 441693,  Ep.Timesteps 516, Score: 1238.62, Avg.Score: 2260.19, Max.Score: 2567.41, Time: 07:00:52 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 2960, Timestep 450534,  Ep.Timesteps 1000, Score: 2427.13, Avg.Score: 2250.93, Max.Score: 2567.41, Time: 07:05:40 \n",
      "Ep. 2970, Timestep 458609,  Ep.Timesteps 1000, Score: 2364.24, Avg.Score: 2207.86, Max.Score: 2548.06, Time: 07:09:50 \n",
      "Ep. 2980, Timestep 468168,  Ep.Timesteps 559, Score: 1388.39, Avg.Score: 2199.64, Max.Score: 2548.06, Time: 07:14:49 \n",
      "Ep. 2990, Timestep 478168,  Ep.Timesteps 1000, Score: 2450.59, Avg.Score: 2229.05, Max.Score: 2548.06, Time: 07:19:12 \n",
      "Ep. 3000, Timestep 488168,  Ep.Timesteps 1000, Score: 2436.02, Avg.Score: 2232.79, Max.Score: 2548.06, Time: 07:24:15 \n",
      "Ep. 3010, Timestep 498168,  Ep.Timesteps 1000, Score: 2547.13, Avg.Score: 2243.20, Max.Score: 2548.06, Time: 07:28:58 \n",
      "Ep. 3020, Timestep 507386,  Ep.Timesteps 1000, Score: 2383.20, Avg.Score: 2229.71, Max.Score: 2548.06, Time: 07:33:51 \n",
      "Ep. 3030, Timestep 517386,  Ep.Timesteps 1000, Score: 2496.95, Avg.Score: 2246.97, Max.Score: 2547.13, Time: 07:38:41 \n",
      "Ep. 3040, Timestep 527386,  Ep.Timesteps 1000, Score: 2512.36, Avg.Score: 2266.41, Max.Score: 2547.13, Time: 07:43:35 \n",
      "Ep. 3050, Timestep 536281,  Ep.Timesteps 1000, Score: 2137.11, Avg.Score: 2287.44, Max.Score: 2547.13, Time: 07:47:45 \n",
      "Ep. 3060, Timestep 546036,  Ep.Timesteps 1000, Score: 2274.54, Avg.Score: 2293.15, Max.Score: 2547.13, Time: 07:52:37 \n",
      "Ep. 3070, Timestep 555542,  Ep.Timesteps 1000, Score: 2401.00, Avg.Score: 2322.49, Max.Score: 2547.13, Time: 07:57:30 \n",
      "Ep. 3080, Timestep 564689,  Ep.Timesteps 1000, Score: 2399.40, Avg.Score: 2312.05, Max.Score: 2560.63, Time: 08:01:31 \n",
      "Ep. 3090, Timestep 573890,  Ep.Timesteps 1000, Score: 2221.51, Avg.Score: 2292.86, Max.Score: 2560.63, Time: 08:05:32 \n",
      "Ep. 3100, Timestep 582092,  Ep.Timesteps 1000, Score: 2418.58, Avg.Score: 2244.33, Max.Score: 2560.63, Time: 08:09:16 \n",
      "Ep. 3110, Timestep 592092,  Ep.Timesteps 1000, Score: 2469.78, Avg.Score: 2245.62, Max.Score: 2560.63, Time: 08:14:13 \n",
      "Ep. 3120, Timestep 592797,  Ep.Timesteps 85, Score: 170.74, Avg.Score: 2041.09, Max.Score: 2560.63, Time: 08:15:09 \n",
      "Ep. 3130, Timestep 595773,  Ep.Timesteps 306, Score: 698.47, Avg.Score: 1861.59, Max.Score: 2560.63, Time: 08:16:50 \n",
      "Ep. 3140, Timestep 604890,  Ep.Timesteps 610, Score: 1375.79, Avg.Score: 1823.97, Max.Score: 2560.63, Time: 08:21:14 \n",
      "Ep. 3150, Timestep 614890,  Ep.Timesteps 1000, Score: 2341.14, Avg.Score: 1836.21, Max.Score: 2560.63, Time: 08:25:51 \n",
      "Ep. 3160, Timestep 624890,  Ep.Timesteps 1000, Score: 2437.79, Avg.Score: 1850.89, Max.Score: 2560.63, Time: 08:32:49 \n",
      "Ep. 3170, Timestep 634890,  Ep.Timesteps 1000, Score: 2165.58, Avg.Score: 1864.31, Max.Score: 2560.63, Time: 08:40:23 \n",
      "Ep. 3180, Timestep 644890,  Ep.Timesteps 1000, Score: 2333.30, Avg.Score: 1874.10, Max.Score: 2548.39, Time: 08:48:16 \n",
      "Ep. 3190, Timestep 654890,  Ep.Timesteps 1000, Score: 2452.77, Avg.Score: 1896.18, Max.Score: 2526.34, Time: 08:55:46 \n",
      "Ep. 3200, Timestep 664657,  Ep.Timesteps 1000, Score: 2383.22, Avg.Score: 1931.76, Max.Score: 2526.34, Time: 09:02:07 \n",
      "Ep. 3210, Timestep 674657,  Ep.Timesteps 1000, Score: 2454.65, Avg.Score: 1931.80, Max.Score: 2543.64, Time: 09:08:46 \n",
      "Ep. 3220, Timestep 683301,  Ep.Timesteps 1000, Score: 2289.17, Avg.Score: 2129.46, Max.Score: 2543.64, Time: 09:14:18 \n",
      "Ep. 3230, Timestep 693301,  Ep.Timesteps 1000, Score: 2486.92, Avg.Score: 2309.02, Max.Score: 2584.44, Time: 09:20:47 \n",
      "Ep. 3240, Timestep 702844,  Ep.Timesteps 1000, Score: 2260.65, Avg.Score: 2328.08, Max.Score: 2584.44, Time: 09:26:54 \n",
      "Ep. 3250, Timestep 712844,  Ep.Timesteps 1000, Score: 2566.67, Avg.Score: 2349.74, Max.Score: 2584.44, Time: 09:33:22 \n",
      "Ep. 3260, Timestep 722844,  Ep.Timesteps 1000, Score: 2503.90, Avg.Score: 2353.09, Max.Score: 2584.44, Time: 09:40:14 \n",
      "Ep. 3270, Timestep 732844,  Ep.Timesteps 1000, Score: 2449.56, Avg.Score: 2362.31, Max.Score: 2584.44, Time: 09:46:46 \n",
      "Ep. 3280, Timestep 742844,  Ep.Timesteps 1000, Score: 2327.61, Avg.Score: 2372.74, Max.Score: 2584.44, Time: 09:53:23 \n",
      "Ep. 3290, Timestep 752390,  Ep.Timesteps 1000, Score: 2467.45, Avg.Score: 2364.03, Max.Score: 2584.44, Time: 10:00:44 \n",
      "Ep. 3300, Timestep 761580,  Ep.Timesteps 1000, Score: 2360.26, Avg.Score: 2359.73, Max.Score: 2584.44, Time: 10:07:02 \n",
      "Ep. 3310, Timestep 771580,  Ep.Timesteps 1000, Score: 2381.03, Avg.Score: 2358.67, Max.Score: 2584.44, Time: 10:13:53 \n",
      "Ep. 3320, Timestep 781580,  Ep.Timesteps 1000, Score: 2479.66, Avg.Score: 2390.62, Max.Score: 2584.44, Time: 10:20:52 \n",
      "Ep. 3330, Timestep 790797,  Ep.Timesteps 217, Score: 482.45, Avg.Score: 2363.52, Max.Score: 2576.28, Time: 10:27:31 \n",
      "Ep. 3340, Timestep 800747,  Ep.Timesteps 1000, Score: 2463.89, Avg.Score: 2380.12, Max.Score: 2576.28, Time: 10:33:53 \n",
      "Ep. 3350, Timestep 810686,  Ep.Timesteps 1000, Score: 2448.25, Avg.Score: 2374.97, Max.Score: 2576.28, Time: 10:40:37 \n",
      "Ep. 3360, Timestep 820686,  Ep.Timesteps 1000, Score: 2469.52, Avg.Score: 2379.79, Max.Score: 2576.28, Time: 10:47:30 \n",
      "Ep. 3370, Timestep 830524,  Ep.Timesteps 1000, Score: 2469.36, Avg.Score: 2382.43, Max.Score: 2586.77, Time: 10:55:32 \n",
      "Ep. 3380, Timestep 840524,  Ep.Timesteps 1000, Score: 2342.37, Avg.Score: 2387.06, Max.Score: 2586.77, Time: 11:04:02 \n",
      "Ep. 3390, Timestep 849363,  Ep.Timesteps 1000, Score: 2452.95, Avg.Score: 2363.81, Max.Score: 2586.77, Time: 11:10:29 \n",
      "Ep. 3400, Timestep 858704,  Ep.Timesteps 1000, Score: 2505.44, Avg.Score: 2361.19, Max.Score: 2586.77, Time: 11:17:10 \n",
      "Ep. 3410, Timestep 868704,  Ep.Timesteps 1000, Score: 2396.25, Avg.Score: 2362.02, Max.Score: 2586.77, Time: 11:25:54 \n",
      "Ep. 3420, Timestep 878704,  Ep.Timesteps 1000, Score: 2559.28, Avg.Score: 2365.59, Max.Score: 2586.77, Time: 11:33:19 \n",
      "Ep. 3430, Timestep 888049,  Ep.Timesteps 1000, Score: 2503.93, Avg.Score: 2380.10, Max.Score: 2586.77, Time: 11:41:33 \n",
      "Ep. 3440, Timestep 897652,  Ep.Timesteps 1000, Score: 2508.58, Avg.Score: 2373.55, Max.Score: 2586.77, Time: 11:49:49 \n",
      "Ep. 3450, Timestep 906450,  Ep.Timesteps 1000, Score: 2359.78, Avg.Score: 2344.47, Max.Score: 2586.77, Time: 11:58:34 \n",
      "Ep. 3460, Timestep 916450,  Ep.Timesteps 1000, Score: 2396.38, Avg.Score: 2344.87, Max.Score: 2586.77, Time: 12:06:30 \n",
      "Ep. 3470, Timestep 925611,  Ep.Timesteps 1000, Score: 2448.82, Avg.Score: 2324.46, Max.Score: 2569.14, Time: 12:12:52 \n",
      "Ep. 3480, Timestep 934807,  Ep.Timesteps 1000, Score: 2577.43, Avg.Score: 2303.79, Max.Score: 2593.73, Time: 12:19:19 \n",
      "Ep. 3490, Timestep 939217,  Ep.Timesteps 758, Score: 1803.61, Avg.Score: 2196.24, Max.Score: 2593.73, Time: 12:23:02 \n",
      "Ep. 3500, Timestep 949217,  Ep.Timesteps 1000, Score: 2488.59, Avg.Score: 2215.88, Max.Score: 2593.73, Time: 12:32:22 \n",
      "Ep. 3510, Timestep 958905,  Ep.Timesteps 1000, Score: 2535.72, Avg.Score: 2214.64, Max.Score: 2593.73, Time: 12:40:30 \n",
      "Ep. 3520, Timestep 968905,  Ep.Timesteps 1000, Score: 2406.35, Avg.Score: 2210.50, Max.Score: 2593.73, Time: 12:47:44 \n",
      "Ep. 3530, Timestep 978255,  Ep.Timesteps 1000, Score: 2478.27, Avg.Score: 2197.83, Max.Score: 2593.73, Time: 12:57:09 \n",
      "Ep. 3540, Timestep 988255,  Ep.Timesteps 1000, Score: 2564.26, Avg.Score: 2210.82, Max.Score: 2593.73, Time: 13:07:27 \n",
      "Ep. 3550, Timestep 997233,  Ep.Timesteps 1000, Score: 2465.73, Avg.Score: 2220.60, Max.Score: 2593.73, Time: 13:15:45 \n",
      "Ep. 3560, Timestep 1007233,  Ep.Timesteps 1000, Score: 2418.03, Avg.Score: 2222.61, Max.Score: 2593.73, Time: 13:25:23 \n",
      "Ep. 3570, Timestep 1017233,  Ep.Timesteps 1000, Score: 2331.31, Avg.Score: 2237.80, Max.Score: 2593.73, Time: 13:35:20 \n",
      "Ep. 3580, Timestep 1026254,  Ep.Timesteps 1000, Score: 2437.13, Avg.Score: 2234.73, Max.Score: 2586.92, Time: 13:44:28 \n",
      "Ep. 3590, Timestep 1034802,  Ep.Timesteps 1000, Score: 2400.89, Avg.Score: 2338.04, Max.Score: 2586.92, Time: 13:53:26 \n",
      "Ep. 3600, Timestep 1044015,  Ep.Timesteps 1000, Score: 2355.29, Avg.Score: 2315.34, Max.Score: 2586.92, Time: 14:02:57 \n",
      "Ep. 3610, Timestep 1053133,  Ep.Timesteps 1000, Score: 2498.93, Avg.Score: 2299.35, Max.Score: 2586.92, Time: 14:11:10 \n",
      "Ep. 3620, Timestep 1062478,  Ep.Timesteps 1000, Score: 2555.65, Avg.Score: 2287.64, Max.Score: 2586.92, Time: 14:18:35 \n",
      "Ep. 3630, Timestep 1072478,  Ep.Timesteps 1000, Score: 2495.07, Avg.Score: 2316.29, Max.Score: 2586.92, Time: 14:27:28 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 3640, Timestep 1082431,  Ep.Timesteps 1000, Score: 2441.29, Avg.Score: 2312.67, Max.Score: 2574.31, Time: 14:36:51 \n",
      "Ep. 3650, Timestep 1092431,  Ep.Timesteps 1000, Score: 2493.98, Avg.Score: 2336.79, Max.Score: 2574.31, Time: 14:46:06 \n",
      "Ep. 3660, Timestep 1098663,  Ep.Timesteps 220, Score: 475.49, Avg.Score: 2240.35, Max.Score: 2574.31, Time: 14:52:47 \n",
      "Ep. 3670, Timestep 1100765,  Ep.Timesteps 124, Score: 260.91, Avg.Score: 2048.30, Max.Score: 2574.31, Time: 14:54:57 \n",
      "Ep. 3680, Timestep 1103126,  Ep.Timesteps 149, Score: 323.91, Avg.Score: 1880.45, Max.Score: 2574.31, Time: 14:57:02 \n",
      "Ep. 3690, Timestep 1105938,  Ep.Timesteps 270, Score: 625.08, Avg.Score: 1739.59, Max.Score: 2574.31, Time: 14:59:39 \n",
      "Ep. 3700, Timestep 1115389,  Ep.Timesteps 1000, Score: 2355.80, Avg.Score: 1739.78, Max.Score: 2574.31, Time: 15:08:11 \n",
      "Ep. 3710, Timestep 1125389,  Ep.Timesteps 1000, Score: 2387.61, Avg.Score: 1744.92, Max.Score: 2574.31, Time: 15:16:52 \n",
      "Ep. 3720, Timestep 1134860,  Ep.Timesteps 1000, Score: 2432.74, Avg.Score: 1747.17, Max.Score: 2537.43, Time: 15:24:27 \n",
      "Ep. 3730, Timestep 1144860,  Ep.Timesteps 1000, Score: 2484.24, Avg.Score: 1739.72, Max.Score: 2535.89, Time: 15:33:22 \n",
      "Ep. 3740, Timestep 1153838,  Ep.Timesteps 1000, Score: 2408.70, Avg.Score: 1710.08, Max.Score: 2551.93, Time: 15:40:05 \n",
      "Ep. 3750, Timestep 1163779,  Ep.Timesteps 1000, Score: 2438.38, Avg.Score: 1714.16, Max.Score: 2552.13, Time: 15:47:27 \n",
      "Ep. 3760, Timestep 1173584,  Ep.Timesteps 805, Score: 2068.71, Avg.Score: 1807.34, Max.Score: 2558.37, Time: 15:55:01 \n",
      "Ep. 3770, Timestep 1183584,  Ep.Timesteps 1000, Score: 2505.55, Avg.Score: 2006.65, Max.Score: 2568.31, Time: 16:02:24 \n",
      "Ep. 3780, Timestep 1193112,  Ep.Timesteps 1000, Score: 2505.27, Avg.Score: 2193.07, Max.Score: 2583.53, Time: 16:09:50 \n",
      "Ep. 3790, Timestep 1202269,  Ep.Timesteps 1000, Score: 2523.44, Avg.Score: 2354.49, Max.Score: 2583.53, Time: 16:16:48 \n",
      "Ep. 3800, Timestep 1212269,  Ep.Timesteps 1000, Score: 2527.27, Avg.Score: 2379.67, Max.Score: 2583.53, Time: 16:24:32 \n",
      "Ep. 3810, Timestep 1221412,  Ep.Timesteps 1000, Score: 2535.85, Avg.Score: 2378.37, Max.Score: 2609.96, Time: 16:31:25 \n",
      "Ep. 3820, Timestep 1230640,  Ep.Timesteps 1000, Score: 2457.54, Avg.Score: 2375.32, Max.Score: 2609.96, Time: 16:38:17 \n",
      "Ep. 3830, Timestep 1240640,  Ep.Timesteps 1000, Score: 2470.37, Avg.Score: 2383.04, Max.Score: 2609.96, Time: 16:45:53 \n",
      "Ep. 3840, Timestep 1250640,  Ep.Timesteps 1000, Score: 2531.39, Avg.Score: 2419.58, Max.Score: 2609.96, Time: 16:53:25 \n",
      "Ep. 3850, Timestep 1260640,  Ep.Timesteps 1000, Score: 2513.19, Avg.Score: 2416.26, Max.Score: 2609.96, Time: 17:00:53 \n",
      "Ep. 3860, Timestep 1269470,  Ep.Timesteps 588, Score: 1517.72, Avg.Score: 2394.65, Max.Score: 2609.96, Time: 17:07:49 \n",
      "Ep. 3870, Timestep 1279320,  Ep.Timesteps 1000, Score: 2452.73, Avg.Score: 2392.20, Max.Score: 2609.96, Time: 17:14:53 \n",
      "Ep. 3880, Timestep 1289320,  Ep.Timesteps 1000, Score: 2456.02, Avg.Score: 2404.35, Max.Score: 2642.33, Time: 17:22:25 \n",
      "Ep. 3890, Timestep 1299320,  Ep.Timesteps 1000, Score: 2523.13, Avg.Score: 2434.09, Max.Score: 2653.20, Time: 17:29:53 \n",
      "Ep. 3900, Timestep 1309053,  Ep.Timesteps 1000, Score: 2598.15, Avg.Score: 2428.55, Max.Score: 2659.89, Time: 17:37:12 \n",
      "Ep. 3910, Timestep 1317618,  Ep.Timesteps 1000, Score: 2262.85, Avg.Score: 2414.07, Max.Score: 2659.89, Time: 17:43:42 \n",
      "Ep. 3920, Timestep 1326958,  Ep.Timesteps 1000, Score: 2100.47, Avg.Score: 2409.13, Max.Score: 2659.89, Time: 17:51:10 \n",
      "Ep. 3930, Timestep 1336958,  Ep.Timesteps 1000, Score: 2411.75, Avg.Score: 2411.89, Max.Score: 2659.89, Time: 17:59:11 \n",
      "Ep. 3940, Timestep 1345556,  Ep.Timesteps 1000, Score: 2466.26, Avg.Score: 2374.28, Max.Score: 2659.89, Time: 18:06:25 \n",
      "Ep. 3950, Timestep 1355556,  Ep.Timesteps 1000, Score: 2497.69, Avg.Score: 2381.51, Max.Score: 2659.89, Time: 18:14:12 \n",
      "Ep. 3960, Timestep 1363277,  Ep.Timesteps 248, Score: 576.65, Avg.Score: 2348.50, Max.Score: 2659.89, Time: 18:20:56 \n",
      "Ep. 3970, Timestep 1371756,  Ep.Timesteps 1000, Score: 2569.43, Avg.Score: 2318.31, Max.Score: 2659.89, Time: 18:26:51 \n",
      "Ep. 3980, Timestep 1380068,  Ep.Timesteps 1000, Score: 2497.28, Avg.Score: 2274.39, Max.Score: 2659.89, Time: 18:33:13 \n",
      "Ep. 3990, Timestep 1388563,  Ep.Timesteps 1000, Score: 2592.92, Avg.Score: 2236.60, Max.Score: 2659.89, Time: 18:39:46 \n",
      "Ep. 4000, Timestep 1398563,  Ep.Timesteps 1000, Score: 2538.32, Avg.Score: 2246.24, Max.Score: 2639.29, Time: 18:47:12 \n",
      "Ep. 4010, Timestep 1408563,  Ep.Timesteps 1000, Score: 2478.78, Avg.Score: 2276.55, Max.Score: 2639.29, Time: 18:55:24 \n",
      "Ep. 4020, Timestep 1416721,  Ep.Timesteps 1000, Score: 2440.36, Avg.Score: 2246.62, Max.Score: 2639.29, Time: 19:01:35 \n",
      "Ep. 4030, Timestep 1425875,  Ep.Timesteps 1000, Score: 2537.93, Avg.Score: 2219.76, Max.Score: 2616.94, Time: 19:08:55 \n",
      "Ep. 4040, Timestep 1435157,  Ep.Timesteps 1000, Score: 2460.15, Avg.Score: 2236.07, Max.Score: 2616.94, Time: 19:16:15 \n",
      "Ep. 4050, Timestep 1444321,  Ep.Timesteps 1000, Score: 2499.45, Avg.Score: 2217.42, Max.Score: 2616.94, Time: 19:23:25 \n",
      "Ep. 4060, Timestep 1453999,  Ep.Timesteps 1000, Score: 2494.96, Avg.Score: 2273.79, Max.Score: 2619.28, Time: 19:30:47 \n",
      "Ep. 4070, Timestep 1463752,  Ep.Timesteps 1000, Score: 2582.70, Avg.Score: 2308.95, Max.Score: 2619.28, Time: 19:38:27 \n",
      "Ep. 4080, Timestep 1473465,  Ep.Timesteps 1000, Score: 2565.19, Avg.Score: 2349.18, Max.Score: 2660.43, Time: 19:45:51 \n",
      "Ep. 4090, Timestep 1482278,  Ep.Timesteps 1000, Score: 2506.19, Avg.Score: 2348.97, Max.Score: 2660.43, Time: 19:52:36 \n",
      "Ep. 4100, Timestep 1491078,  Ep.Timesteps 1000, Score: 2575.86, Avg.Score: 2322.42, Max.Score: 2660.43, Time: 19:59:23 \n",
      "Ep. 4110, Timestep 1499201,  Ep.Timesteps 1000, Score: 2601.29, Avg.Score: 2285.93, Max.Score: 2666.82, Time: 20:05:26 \n",
      "Ep. 4120, Timestep 1507924,  Ep.Timesteps 770, Score: 1997.69, Avg.Score: 2313.57, Max.Score: 2666.82, Time: 20:12:14 \n",
      "Ep. 4130, Timestep 1517924,  Ep.Timesteps 1000, Score: 2531.77, Avg.Score: 2337.00, Max.Score: 2666.82, Time: 20:20:20 \n",
      "Ep. 4140, Timestep 1527593,  Ep.Timesteps 1000, Score: 2454.06, Avg.Score: 2342.95, Max.Score: 2666.82, Time: 20:29:23 \n",
      "Ep. 4150, Timestep 1537593,  Ep.Timesteps 1000, Score: 2371.78, Avg.Score: 2352.36, Max.Score: 2666.82, Time: 20:39:29 \n",
      "Ep. 4160, Timestep 1547376,  Ep.Timesteps 1000, Score: 2571.73, Avg.Score: 2347.10, Max.Score: 2666.82, Time: 20:48:59 \n",
      "Ep. 4170, Timestep 1556589,  Ep.Timesteps 1000, Score: 2676.48, Avg.Score: 2326.50, Max.Score: 2676.48, Time: 20:56:02 \n",
      "Ep. 4180, Timestep 1566493,  Ep.Timesteps 1000, Score: 2580.33, Avg.Score: 2329.30, Max.Score: 2676.48, Time: 21:03:29 \n",
      "Ep. 4190, Timestep 1575691,  Ep.Timesteps 1000, Score: 2493.81, Avg.Score: 2346.16, Max.Score: 2676.48, Time: 21:10:22 \n",
      "Ep. 4200, Timestep 1585691,  Ep.Timesteps 1000, Score: 2557.36, Avg.Score: 2375.35, Max.Score: 2676.48, Time: 21:17:43 \n",
      "Ep. 4210, Timestep 1594989,  Ep.Timesteps 1000, Score: 2431.70, Avg.Score: 2399.45, Max.Score: 2676.48, Time: 21:24:34 \n",
      "Ep. 4220, Timestep 1602757,  Ep.Timesteps 1000, Score: 2470.04, Avg.Score: 2373.77, Max.Score: 2676.48, Time: 21:30:39 \n",
      "Ep. 4230, Timestep 1612622,  Ep.Timesteps 1000, Score: 2506.77, Avg.Score: 2373.58, Max.Score: 2676.48, Time: 21:38:15 \n",
      "Ep. 4240, Timestep 1621636,  Ep.Timesteps 1000, Score: 2523.31, Avg.Score: 2361.11, Max.Score: 2676.48, Time: 21:45:03 \n",
      "Ep. 4250, Timestep 1631636,  Ep.Timesteps 1000, Score: 2481.12, Avg.Score: 2367.35, Max.Score: 2676.48, Time: 21:54:20 \n",
      "Ep. 4260, Timestep 1641127,  Ep.Timesteps 1000, Score: 2615.51, Avg.Score: 2367.04, Max.Score: 2676.48, Time: 22:04:56 \n",
      "Ep. 4270, Timestep 1651127,  Ep.Timesteps 1000, Score: 2476.49, Avg.Score: 2389.73, Max.Score: 2654.41, Time: 22:17:13 \n",
      "Ep. 4280, Timestep 1660389,  Ep.Timesteps 1000, Score: 2521.53, Avg.Score: 2365.77, Max.Score: 2654.41, Time: 22:27:39 \n",
      "Ep. 4290, Timestep 1669065,  Ep.Timesteps 1000, Score: 2568.44, Avg.Score: 2349.42, Max.Score: 2654.41, Time: 22:37:21 \n",
      "Ep. 4300, Timestep 1670177,  Ep.Timesteps 11, Score: 25.08, Avg.Score: 2121.05, Max.Score: 2654.41, Time: 22:39:29 \n",
      "Ep. 4310, Timestep 1670313,  Ep.Timesteps 20, Score: 37.06, Avg.Score: 1890.42, Max.Score: 2654.41, Time: 22:39:39 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 4320, Timestep 1674504,  Ep.Timesteps 1000, Score: 2519.64, Avg.Score: 1798.45, Max.Score: 2630.00, Time: 22:43:42 \n",
      "Ep. 4330, Timestep 1682755,  Ep.Timesteps 1000, Score: 2619.47, Avg.Score: 1753.56, Max.Score: 2629.71, Time: 22:53:20 \n",
      "Ep. 4340, Timestep 1690883,  Ep.Timesteps 1000, Score: 2565.93, Avg.Score: 1735.34, Max.Score: 2629.71, Time: 23:02:10 \n",
      "Ep. 4350, Timestep 1700399,  Ep.Timesteps 1000, Score: 2370.26, Avg.Score: 1721.80, Max.Score: 2629.71, Time: 23:14:36 \n",
      "Ep. 4360, Timestep 1707878,  Ep.Timesteps 122, Score: 237.94, Avg.Score: 1664.45, Max.Score: 2629.71, Time: 23:25:28 \n",
      "Ep. 4370, Timestep 1710385,  Ep.Timesteps 55, Score: 90.32, Avg.Score: 1468.39, Max.Score: 2629.71, Time: 23:27:55 \n",
      "Ep. 4380, Timestep 1715893,  Ep.Timesteps 48, Score: 29.03, Avg.Score: 1370.46, Max.Score: 2629.71, Time: 23:32:57 \n",
      "Ep. 4390, Timestep 1720453,  Ep.Timesteps 1000, Score: 2530.23, Avg.Score: 1260.47, Max.Score: 2629.71, Time: 23:36:09 \n",
      "Ep. 4400, Timestep 1726021,  Ep.Timesteps 1000, Score: 2376.56, Avg.Score: 1369.51, Max.Score: 2632.20, Time: 23:40:58 \n",
      "Ep. 4410, Timestep 1732742,  Ep.Timesteps 155, Score: 337.08, Avg.Score: 1529.20, Max.Score: 2632.20, Time: 23:47:36 \n",
      "Ep. 4420, Timestep 1741165,  Ep.Timesteps 1000, Score: 2362.80, Avg.Score: 1629.98, Max.Score: 2632.20, Time: 23:54:11 \n",
      "Ep. 4430, Timestep 1750543,  Ep.Timesteps 1000, Score: 2502.92, Avg.Score: 1658.46, Max.Score: 2651.61, Time: 24:01:51 \n",
      "Ep. 4440, Timestep 1760543,  Ep.Timesteps 1000, Score: 2427.42, Avg.Score: 1700.30, Max.Score: 2651.61, Time: 24:09:44 \n",
      "Ep. 4450, Timestep 1770543,  Ep.Timesteps 1000, Score: 2480.16, Avg.Score: 1707.63, Max.Score: 2651.61, Time: 24:17:38 \n",
      "Ep. 4460, Timestep 1780166,  Ep.Timesteps 1000, Score: 2515.97, Avg.Score: 1759.99, Max.Score: 2651.61, Time: 24:25:07 \n",
      "Ep. 4470, Timestep 1790166,  Ep.Timesteps 1000, Score: 2361.53, Avg.Score: 1951.30, Max.Score: 2651.61, Time: 24:33:21 \n",
      "Ep. 4480, Timestep 1800032,  Ep.Timesteps 1000, Score: 2494.46, Avg.Score: 2066.43, Max.Score: 2651.61, Time: 24:41:25 \n",
      "Ep. 4490, Timestep 1810032,  Ep.Timesteps 1000, Score: 2340.21, Avg.Score: 2205.26, Max.Score: 2651.61, Time: 24:49:39 \n",
      "Ep. 4500, Timestep 1820032,  Ep.Timesteps 1000, Score: 2515.83, Avg.Score: 2315.60, Max.Score: 2651.61, Time: 24:57:22 \n",
      "Ep. 4510, Timestep 1828291,  Ep.Timesteps 1000, Score: 2555.32, Avg.Score: 2361.13, Max.Score: 2651.61, Time: 25:04:01 \n",
      "Ep. 4520, Timestep 1837990,  Ep.Timesteps 1000, Score: 2504.67, Avg.Score: 2398.55, Max.Score: 2651.61, Time: 25:13:08 \n",
      "Ep. 4530, Timestep 1847300,  Ep.Timesteps 1000, Score: 2473.01, Avg.Score: 2398.70, Max.Score: 2580.12, Time: 25:22:23 \n",
      "Ep. 4540, Timestep 1857300,  Ep.Timesteps 1000, Score: 2523.14, Avg.Score: 2402.92, Max.Score: 2591.54, Time: 25:30:36 \n",
      "Ep. 4550, Timestep 1867300,  Ep.Timesteps 1000, Score: 2540.14, Avg.Score: 2413.49, Max.Score: 2625.06, Time: 25:38:27 \n",
      "Ep. 4560, Timestep 1877300,  Ep.Timesteps 1000, Score: 2562.84, Avg.Score: 2427.52, Max.Score: 2625.06, Time: 25:46:21 \n",
      "Ep. 4570, Timestep 1887093,  Ep.Timesteps 1000, Score: 2506.56, Avg.Score: 2428.75, Max.Score: 2625.06, Time: 25:53:56 \n",
      "Ep. 4580, Timestep 1897093,  Ep.Timesteps 1000, Score: 2588.38, Avg.Score: 2437.35, Max.Score: 2625.06, Time: 26:02:34 \n",
      "Ep. 4590, Timestep 1905520,  Ep.Timesteps 1000, Score: 2506.91, Avg.Score: 2401.56, Max.Score: 2625.06, Time: 26:09:22 \n",
      "Ep. 4600, Timestep 1915520,  Ep.Timesteps 1000, Score: 2558.67, Avg.Score: 2409.26, Max.Score: 2625.06, Time: 26:17:18 \n",
      "Ep. 4610, Timestep 1923409,  Ep.Timesteps 1000, Score: 2456.53, Avg.Score: 2400.45, Max.Score: 2625.06, Time: 26:23:31 \n",
      "Ep. 4620, Timestep 1933409,  Ep.Timesteps 1000, Score: 2466.67, Avg.Score: 2411.55, Max.Score: 2625.06, Time: 26:31:33 \n",
      "Ep. 4630, Timestep 1943409,  Ep.Timesteps 1000, Score: 2528.82, Avg.Score: 2433.17, Max.Score: 2625.06, Time: 26:39:35 \n",
      "Ep. 4640, Timestep 1953409,  Ep.Timesteps 1000, Score: 2584.70, Avg.Score: 2437.45, Max.Score: 2642.73, Time: 26:47:20 \n",
      "Ep. 4650, Timestep 1963409,  Ep.Timesteps 1000, Score: 2412.23, Avg.Score: 2431.79, Max.Score: 2642.73, Time: 26:55:43 \n",
      "Ep. 4660, Timestep 1973409,  Ep.Timesteps 1000, Score: 2489.78, Avg.Score: 2433.67, Max.Score: 2642.73, Time: 27:03:42 \n",
      "Ep. 4670, Timestep 1982482,  Ep.Timesteps 1000, Score: 2499.70, Avg.Score: 2416.02, Max.Score: 2642.73, Time: 27:11:00 \n",
      "Ep. 4680, Timestep 1992482,  Ep.Timesteps 1000, Score: 2456.54, Avg.Score: 2410.27, Max.Score: 2642.73, Time: 27:19:00 \n",
      "Ep. 4690, Timestep 2002482,  Ep.Timesteps 1000, Score: 2503.27, Avg.Score: 2448.43, Max.Score: 2642.73, Time: 27:27:15 \n",
      "Ep. 4700, Timestep 2012482,  Ep.Timesteps 1000, Score: 2448.04, Avg.Score: 2438.42, Max.Score: 2642.73, Time: 27:35:15 \n",
      "Ep. 4710, Timestep 2021252,  Ep.Timesteps 1000, Score: 2466.30, Avg.Score: 2440.05, Max.Score: 2642.73, Time: 27:42:22 \n",
      "Ep. 4720, Timestep 2030437,  Ep.Timesteps 1000, Score: 2469.74, Avg.Score: 2357.61, Max.Score: 2642.73, Time: 27:49:33 \n",
      "Ep. 4730, Timestep 2033678,  Ep.Timesteps 27, Score: 29.75, Avg.Score: 2178.60, Max.Score: 2642.73, Time: 27:53:31 \n",
      "Ep. 4740, Timestep 2035186,  Ep.Timesteps 55, Score: 88.54, Avg.Score: 1954.62, Max.Score: 2634.29, Time: 27:55:08 \n",
      "Ep. 4750, Timestep 2035933,  Ep.Timesteps 54, Score: 79.68, Avg.Score: 1716.72, Max.Score: 2634.29, Time: 27:56:00 \n",
      "Ep. 4760, Timestep 2036546,  Ep.Timesteps 55, Score: 79.55, Avg.Score: 1473.03, Max.Score: 2622.60, Time: 27:56:45 \n",
      "Ep. 4770, Timestep 2037091,  Ep.Timesteps 51, Score: 73.31, Avg.Score: 1250.41, Max.Score: 2585.62, Time: 27:57:24 \n",
      "Ep. 4780, Timestep 2037715,  Ep.Timesteps 66, Score: 91.91, Avg.Score: 1011.47, Max.Score: 2585.62, Time: 27:58:12 \n",
      "Ep. 4790, Timestep 2038378,  Ep.Timesteps 90, Score: 135.83, Avg.Score: 771.32, Max.Score: 2585.62, Time: 27:59:05 \n",
      "Ep. 4800, Timestep 2039315,  Ep.Timesteps 112, Score: 193.62, Avg.Score: 542.26, Max.Score: 2585.62, Time: 28:00:15 \n",
      "Ep. 4810, Timestep 2040405,  Ep.Timesteps 160, Score: 296.59, Avg.Score: 360.07, Max.Score: 2539.23, Time: 28:01:28 \n",
      "Ep. 4820, Timestep 2042698,  Ep.Timesteps 411, Score: 658.30, Avg.Score: 225.58, Max.Score: 2530.26, Time: 28:03:52 \n",
      "Ep. 4830, Timestep 2044579,  Ep.Timesteps 236, Score: 464.27, Avg.Score: 183.66, Max.Score: 2521.97, Time: 28:06:25 \n",
      "Ep. 4840, Timestep 2047127,  Ep.Timesteps 237, Score: 520.15, Avg.Score: 203.10, Max.Score: 1921.66, Time: 28:09:36 \n",
      "Ep. 4850, Timestep 2049719,  Ep.Timesteps 245, Score: 522.50, Avg.Score: 248.17, Max.Score: 1921.66, Time: 28:12:36 \n",
      "Ep. 4860, Timestep 2052633,  Ep.Timesteps 178, Score: 337.20, Avg.Score: 300.15, Max.Score: 1921.66, Time: 28:15:48 \n",
      "Ep. 4870, Timestep 2057447,  Ep.Timesteps 425, Score: 805.71, Avg.Score: 390.09, Max.Score: 2037.14, Time: 28:20:35 \n",
      "Ep. 4880, Timestep 2064224,  Ep.Timesteps 1000, Score: 742.35, Avg.Score: 458.59, Max.Score: 2136.47, Time: 28:27:54 \n",
      "Ep. 4890, Timestep 2067142,  Ep.Timesteps 73, Score: 130.69, Avg.Score: 479.49, Max.Score: 2136.47, Time: 28:32:09 \n",
      "Ep. 4900, Timestep 2068242,  Ep.Timesteps 73, Score: 128.39, Avg.Score: 485.44, Max.Score: 2136.47, Time: 28:33:23 \n",
      "Ep. 4910, Timestep 2075543,  Ep.Timesteps 137, Score: 266.93, Avg.Score: 603.99, Max.Score: 2163.19, Time: 28:41:56 \n",
      "Ep. 4920, Timestep 2082582,  Ep.Timesteps 1000, Score: 2464.89, Avg.Score: 730.81, Max.Score: 2464.89, Time: 28:49:19 \n",
      "Ep. 4930, Timestep 2092582,  Ep.Timesteps 1000, Score: 2556.20, Avg.Score: 935.97, Max.Score: 2556.20, Time: 29:00:59 \n",
      "Ep. 4940, Timestep 2101983,  Ep.Timesteps 1000, Score: 2432.25, Avg.Score: 1112.69, Max.Score: 2556.20, Time: 29:13:59 \n",
      "Ep. 4950, Timestep 2111983,  Ep.Timesteps 1000, Score: 2301.93, Avg.Score: 1301.04, Max.Score: 2556.20, Time: 29:26:10 \n",
      "Ep. 4960, Timestep 2121983,  Ep.Timesteps 1000, Score: 2453.86, Avg.Score: 1486.58, Max.Score: 2557.08, Time: 29:38:56 \n",
      "Ep. 4970, Timestep 2131983,  Ep.Timesteps 1000, Score: 2389.39, Avg.Score: 1633.60, Max.Score: 2557.08, Time: 29:50:05 \n",
      "Ep. 4980, Timestep 2141581,  Ep.Timesteps 598, Score: 1429.07, Avg.Score: 1791.50, Max.Score: 2573.66, Time: 30:00:36 \n",
      "Ep. 4990, Timestep 2151581,  Ep.Timesteps 1000, Score: 2530.54, Avg.Score: 2015.14, Max.Score: 2658.13, Time: 30:10:49 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 5000, Timestep 2161581,  Ep.Timesteps 1000, Score: 2523.00, Avg.Score: 2239.47, Max.Score: 2658.13, Time: 30:22:01 \n",
      "Ep. 5010, Timestep 2171581,  Ep.Timesteps 1000, Score: 2513.63, Avg.Score: 2350.10, Max.Score: 2658.13, Time: 30:33:06 \n",
      "Ep. 5020, Timestep 2181391,  Ep.Timesteps 810, Score: 2027.05, Avg.Score: 2427.09, Max.Score: 2658.13, Time: 30:43:29 \n",
      "Ep. 5030, Timestep 2191391,  Ep.Timesteps 1000, Score: 2509.74, Avg.Score: 2433.87, Max.Score: 2658.13, Time: 30:53:20 \n",
      "Ep. 5040, Timestep 2201391,  Ep.Timesteps 1000, Score: 2423.03, Avg.Score: 2446.28, Max.Score: 2658.13, Time: 31:03:58 \n",
      "Ep. 5050, Timestep 2211391,  Ep.Timesteps 1000, Score: 2393.46, Avg.Score: 2441.88, Max.Score: 2658.13, Time: 31:13:27 \n",
      "Ep. 5060, Timestep 2221391,  Ep.Timesteps 1000, Score: 2374.02, Avg.Score: 2431.17, Max.Score: 2658.13, Time: 31:22:34 \n",
      "Ep. 5070, Timestep 2231391,  Ep.Timesteps 1000, Score: 2378.18, Avg.Score: 2423.25, Max.Score: 2658.13, Time: 31:31:31 \n",
      "Ep. 5080, Timestep 2241391,  Ep.Timesteps 1000, Score: 2316.21, Avg.Score: 2428.60, Max.Score: 2658.13, Time: 31:40:31 \n",
      "Ep. 5090, Timestep 2251391,  Ep.Timesteps 1000, Score: 2456.78, Avg.Score: 2419.29, Max.Score: 2573.77, Time: 31:49:26 \n",
      "Ep. 5100, Timestep 2261391,  Ep.Timesteps 1000, Score: 2312.50, Avg.Score: 2408.90, Max.Score: 2573.77, Time: 31:59:42 \n",
      "Ep. 5110, Timestep 2271344,  Ep.Timesteps 1000, Score: 2550.03, Avg.Score: 2402.20, Max.Score: 2591.58, Time: 32:09:01 \n",
      "Ep. 5120, Timestep 2281344,  Ep.Timesteps 1000, Score: 2533.34, Avg.Score: 2411.17, Max.Score: 2594.16, Time: 32:18:34 \n",
      "Ep. 5130, Timestep 2291344,  Ep.Timesteps 1000, Score: 2470.40, Avg.Score: 2416.83, Max.Score: 2598.34, Time: 32:28:17 \n",
      "Ep. 5140, Timestep 2301344,  Ep.Timesteps 1000, Score: 2556.82, Avg.Score: 2418.77, Max.Score: 2598.34, Time: 32:39:04 \n",
      "Ep. 5150, Timestep 2311344,  Ep.Timesteps 1000, Score: 2484.83, Avg.Score: 2426.97, Max.Score: 2598.34, Time: 32:49:47 \n",
      "Ep. 5160, Timestep 2321344,  Ep.Timesteps 1000, Score: 2536.82, Avg.Score: 2443.83, Max.Score: 2628.15, Time: 33:00:49 \n",
      "Ep. 5170, Timestep 2331344,  Ep.Timesteps 1000, Score: 2551.21, Avg.Score: 2463.99, Max.Score: 2628.15, Time: 33:12:03 \n",
      "Ep. 5180, Timestep 2340962,  Ep.Timesteps 1000, Score: 2571.93, Avg.Score: 2465.59, Max.Score: 2628.15, Time: 33:21:19 \n",
      "Ep. 5190, Timestep 2348890,  Ep.Timesteps 1000, Score: 2601.60, Avg.Score: 2420.62, Max.Score: 2628.15, Time: 33:29:32 \n",
      "Ep. 5200, Timestep 2358505,  Ep.Timesteps 1000, Score: 2500.51, Avg.Score: 2426.96, Max.Score: 2628.15, Time: 33:40:10 \n",
      "Ep. 5210, Timestep 2367460,  Ep.Timesteps 1000, Score: 2421.31, Avg.Score: 2408.78, Max.Score: 2628.15, Time: 33:50:46 \n",
      "Ep. 5220, Timestep 2377453,  Ep.Timesteps 993, Score: 2485.99, Avg.Score: 2404.98, Max.Score: 2628.15, Time: 34:02:00 \n",
      "Ep. 5230, Timestep 2387453,  Ep.Timesteps 1000, Score: 2428.58, Avg.Score: 2402.15, Max.Score: 2628.15, Time: 34:12:29 \n",
      "Ep. 5240, Timestep 2397453,  Ep.Timesteps 1000, Score: 2475.86, Avg.Score: 2407.77, Max.Score: 2628.15, Time: 34:21:21 \n",
      "Ep. 5250, Timestep 2407453,  Ep.Timesteps 1000, Score: 2513.36, Avg.Score: 2410.06, Max.Score: 2628.15, Time: 34:31:33 \n",
      "Ep. 5260, Timestep 2414991,  Ep.Timesteps 1000, Score: 2587.44, Avg.Score: 2345.39, Max.Score: 2618.96, Time: 34:38:04 \n",
      "Ep. 5270, Timestep 2424991,  Ep.Timesteps 1000, Score: 2479.95, Avg.Score: 2338.67, Max.Score: 2601.60, Time: 34:48:35 \n",
      "Ep. 5280, Timestep 2434991,  Ep.Timesteps 1000, Score: 2530.69, Avg.Score: 2349.57, Max.Score: 2603.54, Time: 34:59:03 \n",
      "Ep. 5290, Timestep 2444991,  Ep.Timesteps 1000, Score: 2461.12, Avg.Score: 2399.71, Max.Score: 2603.54, Time: 35:10:08 \n",
      "Ep. 5300, Timestep 2446462,  Ep.Timesteps 47, Score: -4.72, Avg.Score: 2185.04, Max.Score: 2603.54, Time: 35:12:54 \n",
      "Ep. 5310, Timestep 2446827,  Ep.Timesteps 37, Score: -0.74, Avg.Score: 1961.74, Max.Score: 2603.54, Time: 35:13:23 \n",
      "Ep. 5320, Timestep 2447144,  Ep.Timesteps 22, Score: 16.95, Avg.Score: 1716.26, Max.Score: 2603.54, Time: 35:13:50 \n",
      "Ep. 5330, Timestep 2451193,  Ep.Timesteps 1000, Score: 2347.17, Avg.Score: 1560.80, Max.Score: 2603.54, Time: 35:17:04 \n",
      "Ep. 5340, Timestep 2460879,  Ep.Timesteps 1000, Score: 2418.97, Avg.Score: 1549.77, Max.Score: 2606.76, Time: 35:26:56 \n",
      "Ep. 5350, Timestep 2470879,  Ep.Timesteps 1000, Score: 2417.09, Avg.Score: 1547.47, Max.Score: 2606.76, Time: 35:35:58 \n",
      "Ep. 5360, Timestep 2480879,  Ep.Timesteps 1000, Score: 2490.82, Avg.Score: 1611.61, Max.Score: 2647.00, Time: 35:45:49 \n",
      "Ep. 5370, Timestep 2490879,  Ep.Timesteps 1000, Score: 2510.47, Avg.Score: 1610.69, Max.Score: 2647.00, Time: 35:55:56 \n",
      "Ep. 5380, Timestep 2500879,  Ep.Timesteps 1000, Score: 2492.26, Avg.Score: 1611.40, Max.Score: 2667.61, Time: 36:05:29 \n",
      "Ep. 5390, Timestep 2510876,  Ep.Timesteps 1000, Score: 2614.13, Avg.Score: 1616.18, Max.Score: 2667.61, Time: 36:14:36 \n",
      "Ep. 5400, Timestep 2520796,  Ep.Timesteps 1000, Score: 2615.48, Avg.Score: 1848.83, Max.Score: 2724.84, Time: 36:23:16 \n",
      "Ep. 5410, Timestep 2530428,  Ep.Timesteps 1000, Score: 2663.14, Avg.Score: 2101.26, Max.Score: 2724.84, Time: 36:32:07 \n",
      "Ep. 5420, Timestep 2539501,  Ep.Timesteps 697, Score: 1850.03, Avg.Score: 2342.38, Max.Score: 2765.72, Time: 36:41:54 \n",
      "Ep. 5430, Timestep 2548388,  Ep.Timesteps 1000, Score: 2654.87, Avg.Score: 2483.89, Max.Score: 2780.82, Time: 36:51:31 \n",
      "Ep. 5438, Timestep 2556388,  Ep.Timesteps 1000, Score: 2531.55, Avg.Score: 2500.79, Max.Score: 2780.82, Time: 36:59:22 \n",
      "Environment solved with Average Score:  2500.789949571582\n"
     ]
    }
   ],
   "source": [
    "# Twin Delayed Deep Deterministic (TD3) policy gradient algorithm\n",
    "def twin_ddd_train(n_episodes=15000, save_every=10, print_env=10):\n",
    "\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = []    \n",
    "\n",
    "    time_start = time.time()                    # Init start time\n",
    "    replay_buf = ReplayBuffer()                 # Init ReplayBuffer\n",
    "    \n",
    "    timestep_after_last_save = 0\n",
    "    total_timesteps = 0\n",
    "    \n",
    "    low = env.action_space.low\n",
    "    high = env.action_space.high\n",
    "    \n",
    "    print('Low in action space: ', low, ', High: ', high, ', Action_dim: ', action_dim)\n",
    "            \n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        \n",
    "        timestep = 0\n",
    "        total_reward = 0\n",
    "        \n",
    "        # Reset environment\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        \n",
    "        while True:\n",
    "            \n",
    "            # Select action randomly or according to policy\n",
    "            if total_timesteps < start_timestep:\n",
    "                action = env.action_space.sample()\n",
    "            else:\n",
    "                action = agent.select_action(np.array(state))\n",
    "                if std_noise != 0: \n",
    "                    shift_action = np.random.normal(0, std_noise, size=action_dim)\n",
    "                    action = (action + shift_action).clip(low, high)\n",
    "            \n",
    "            # Perform action\n",
    "            new_state, reward, done, _ = env.step(action) \n",
    "            done_bool = 0 if timestep + 1 == env._max_episode_steps else float(done)\n",
    "            total_reward += reward                          # full episode reward\n",
    "\n",
    "            # Store every timestep in replay buffer\n",
    "            replay_buf.add((state, new_state, action, reward, done_bool))\n",
    "            state = new_state\n",
    "\n",
    "            timestep += 1     \n",
    "            total_timesteps += 1\n",
    "            timestep_after_last_save += 1\n",
    "\n",
    "            if done:                                       # done ?\n",
    "                break                                      # save score\n",
    "\n",
    "        scores_deque.append(total_reward)\n",
    "        scores_array.append(total_reward)\n",
    "\n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "\n",
    "        max_score = np.max(scores_deque)\n",
    "\n",
    "        # train_by_episode(time_start, i_episode) \n",
    "        s = (int)(time.time() - time_start)\n",
    "        if i_episode % print_env == 0 or (len(scores_deque) == 100 and avg_score > threshold):\n",
    "            print('Ep. {}, Timestep {},  Ep.Timesteps {}, Score: {:.2f}, Avg.Score: {:.2f}, Max.Score: {:.2f}, Time: {:02}:{:02}:{:02} '\\\n",
    "                .format(i_episode, total_timesteps, timestep, \\\n",
    "                        total_reward, avg_score, max_score, s//3600, s%3600//60, s%60))     \n",
    "\n",
    "        agent.train(replay_buf, timestep)\n",
    "\n",
    "        # Save episode if more than save_every=5000 timesteps\n",
    "        if timestep_after_last_save >= save_every and i_episode > 0:\n",
    "\n",
    "            timestep_after_last_save %= save_every            \n",
    "            save(agent, 'chpnt_interm', 'dir_chk_002')  \n",
    "        \n",
    "        if len(scores_deque) == 100 and avg_score >= threshold:\n",
    "            print('Environment solved with Average Score: ',  avg_score )\n",
    "            break \n",
    "\n",
    "    return scores_array, avg_scores_array\n",
    "\n",
    "scores, avg_scores = twin_ddd_train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(agent, 'chpnt_ts2500', 'dir_chk_002')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  5438 , len of avg_scores:  5438\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAAEGCAYAAACpaqUaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydd5hU1d3HP2faVnZpSy+rwApLRxQbig0bsWvUxJKYQCyxxBITE31jNBqNr3lN7ImK2MUSKypYsBEB6UWKAtKXunX6ef+YO7MzO332zu7s7u/zPPtw59xzzz0zO+z53l87SmuNIAiCIAhCrmNp7QkIgiAIgiCkgogWQRAEQRDaBCJaBEEQBEFoE4hoEQRBEAShTSCiRRAEQRCENoGttSeQDbp3767Ly8tbexqCIAhtioULF+7SWpe19jwEIR7tUrSUl5ezYMGC1p6GIAhCm0IptbG15yAIiRD3kCAIgiAIbQIRLYIgCIIgtAlEtAiCIAiC0CZolzEtgiAIQu6wcOHCHjab7V/ACORhWYiPH1ju9Xp/cfDBB++M1UFEiyAIgpBVbDbbv3r16jWsrKxsr8VikQ3vhJj4/X5VVVVVuX379n8Bp8fqI4pXEARByDYjysrKqkWwCImwWCy6rKxsPwGLXOw+LTgfQRAEoWNiEcEipILxPYmrTUS0CIIgtAN2VDuZvXJHa09DELKKiBZBEIR2wIS/zOEXz0hRzUT89re/7TV48ODhFRUVlUOHDq386KOPilp7TkJ6SCCuIAhCO0JrjVKqtaeRc8yePbvo/fff77xs2bKVBQUFetu2bTaXy5XxB+XxeLDb7WZOUUgBsbQIgiC0I7REjsRky5Yt9q5du3oLCgo0QO/evb3l5eWeTz/9tHDs2LFDDzrooMqRI0cO27t3r6W+vl6de+655RUVFZXDhg2rfOuttzoBPPjgg91OOeWUA4877rjBEydOrAD44x//2HPEiBHDKioqKq+//vo+rfkeOwJiaREEQWjjvLdsW+j4o9U7OaGyZyvOJjE3zVzSf832mkIzx6zo1an+vnNH/5Coz5lnnll999139ykvLx9x1FFHVV944YV7jj/++Lqf/OQng5577rn1xxxzTP2ePXssxcXF/jvvvLMnwJo1a1YuWrQo/9RTTx2yfv365QDffPNN8dKlS1f07NnT99prr5WsW7cuf+nSpau01pxwwgmD33vvveJTTjml1sz3JzQilhZBEIQ2zhXPfRM6lriW2JSWlvqXL1++8p///OfGsrIy76WXXjro/vvvL+vRo4fnmGOOqQfo2rWr32638+WXXxZfcskluwHGjh3r7NOnj3vZsmX5ABMnTqzu2bOnD2DWrFklc+fOLamsrKwcPnx45fr16/NXr16d33rvsv0jlhZBEAShxUhmEckmNpuNKVOm1EyZMqVm1KhRDY8++miZUirKoaYT+NgKCwv94f2uu+66bTfddNOuLE1ZaIJYWgRBaFP4/Bqvz5+8oyCEsWTJkrxly5blBV8vWrSoYMiQIc4dO3Y4Pv3000KAvXv3WjweD0cddVTts88+2xVg6dKledu2bXOMGjXK2XTMU045pXrGjBnd9+/fbwH4/vvv7Vu2bBFjQBaRD1cQhJxmR7WTTXvqOaS8KwDH3f8Jm/bU8/3dp7XyzHKDN5dsbe0ptAmqq6ut11xzzYDq6mqr1WrV5eXlrunTp29cs2bNrmuuuWaA0+m05Ofn++fOnbvm5ptv3nnxxRcPrKioqLRarTz22GMbggG84Zx99tnVK1asyD/kkEOGQsAK89xzz33ft29fb8u/w46BSmQGa6uMHz9eL1ggfl1ByEU++XYnlz01nzk3HMOgsuK4/X7YU8/Eez8Ovd5wT0CklN/yDgDf/eVUnvziey6aMIBCR/LnL7fXz9LN+xhviJ+WZGeNk19MX8ATl4ynZ4l5IQ87qp1M+MucqPbgZ5UuSqmFWuvxzZ1XU5YsWbJh9OjR4kIRUmLJkiXdR48eXR7rnLiHBEFoMdZX1XLZU/MBeOqL7/lodfwKrvO+2x3x+tWFmyNe/+zp+dz5zirunfVtSvf+y7urOPfRr1i9vTql/v9ZvIUXv96E0+NLqX8iXvjvDyzdvJ/n5m1s9lhBlvywL6ZgEYT2jIgWQRBajJ8/PT90/Oy8Tfz86fgW0aYF0m54ZUnE60/XVAHw9Jcboq69eeYS3li0JfS6/JZ3Qv321Lmj+i/5YR/Lt+wPvfb6/Fz74mJueW0ZQ/84iwZ384SLxXgrviSW7Z01Tv74xnLc3uQxO3NW72zWnAShLSKiRRCErLJo01627w/EMPr80Yu236+ZtXw75be8w65aV6j9xiYiJRHPztvIoN+/y0/+NQ+Alxds5rqXFsfOAonRdMZDXzDlH58D8O/Pv2fwre9FnP/Ny4tTnkssLIZqSRY/fMdbK5kxbyMVf3gvYT+fX7OvPlp8CUJ7R0SLIAhR7Kp1sTeGRSITznr4S46572M27a5n896GqPO3vLaUXz27EIDxd87mzSVbOeOfn8cca+HGPTHb//DGcnx+zRfrIl1KH6VpjTj/0a/489sro9rfW749rXGaYjVES7wYwuVb9nP6Pz8PiTuA61+KFEoXPTGPq4x6LPfOWs0zX5nnahKEtoKIFkFox3y/q45nvtoQ9/ys5du4+71VUe3j75zN2D9/CARcJVc9902E+yQVZi7cHFqEXV4/R9/3ccx+Ly+IjFV5f8V2lmyOfa8rw4qopcLSOOOE882mvaHjrzfEFkUAc9dU8WyGMSlB95A/hmhZX1XLlH98ztLN+1mwsXEur4e5t6Z/uYEv1+/mHaPybXNFlCC0VSTlWRByiP31HlZtr+awA7uZMt45j3zJnjo3g8qKOfzAbvz03/9lxdZqltw+GYBfPRsQATdNPggNbNxdx+AenULXr9tZAyjeWbaNd5ZtC2Wl3Pf+alZvq+Hflx0S876rt1dz4ytL6Nu5IO05O6zxn6V2VLvingsSbq3Y3+CJOl/r8vLxtzv52VPzo84l4pInvwbgp4cNTOs6gL+8uxqAJz77nuOG9mRor050KXLwwYrtTJ2xMO51t7y6lP5dC7nv/dSCjQWhvSOWFkFoJXbWOPnP4i14fH6Ovvdj3l+xnV88M58LHp+XcuDnaQ9+xt8SLGh7jbiHn/zrvzz5xfd8uX43+xs8URkxL8z/gSG3vscJ/zuXbfsbXTgrtlaTZ2v8M/FdVS2zV+7goY/Xxw0EnbumipP//hkAW/ZFu4OSYWnmDsWH3d2YURMrSHfqjIVpC5ZM8Ps1/zd7bcilE+TCJ+aFrFi/fmFRwjFenP9DlGAxI5upo/LMM890VkodvGjRopwotb99+3brhAkTKgoLC8decsklA8LPffbZZ4UVFRWVAwYMGHHZZZf19/sDAVE7duywHnHEEUMGDhw44ogjjhhSVVVlNWMuxxxzzOBdu3Y1a6y3336707HHHjvYjPnEQ0SLILQgPr9mtxFseur/fca1Ly5m894GNu2pZ9qMhczfEHAPuH1+Nu2up/yWd5izKpAWvGDDHspveYcJf5mN3whoXbG1mn9+vC5mrMTX3++J2PE33FVy76xv+c/iRvfDH99YHjoOt05c++JiXglLNf71C4si9rbZX++hztVYR2vRpr0hi0SmfL8rt/eaCw8WTsQna3bywOw1IZeOWdz/gVhdMuXFF1/sOm7cuNoZM2a0fLGeGBQWFuo77rhj6//8z/9sbnruyiuvHPjwww9v3LBhw/Lvvvsuf+bMmSUAt99+e+9JkybVbNy4cfmkSZNqbrvttl5mzOXTTz9d171795xXxFkTLUqp/kqpj5VSq5RSK5RS1xrt/6OU2qKUWmz8nBp2ze+UUuuUUt8qpU4Kaz/ZaFunlLolW3MWhGzz11mrOfjO2eyrd7OrNmAFCbdkBPH6/CwzYkgun76ARz5Zz7mPfgUEXCR3NAkWDU/KWbezhs/X7uL8x76K6BNeOfXJL77nha83xZzj9CbWiQfnrA0dr9gaWeNk9B0fMPz294HAYn7Ww1/GHDMdvtm0r9ljhPPFOnNrmo2/c3bCvWmCJEtbzjQj6YnPvmfTnvqMru3I7N+/37JgwYLip556asPrr7/eJdh+2mmnHfjSSy+VBl+fc8455U8//XTnmpoay6mnnnpgRUVF5WmnnXbgqFGjhs6dOzdqd+r//Oc/nYYNG1ZZUVFRed5555U3NDQogL59+468/vrr+1RWVg6rqKiojGXdKSkp8Z900km1+fn5EV+WjRs32mtray0nnHBCncVi4Sc/+cnuN954owvArFmzOk+bNm03wLRp03a/9957XZqO6/V6mTZtWr8RI0YMq6ioqLzvvvu6Q8ASMn78+INOPPHEQYMGDRp+0UUXDfD5Ajqlb9++I7dt22arrq62TJo0afBBBx1UOWTIkOFPPPFEl0Tvc+bMmSUHHHDA8IMPPvigmTNndg7Oobq62nLeeeeVjxgxYtiwYcMqn3322c4ACxYsyB85cuSwoUOHVlZUVFSGb62QCtmMafECN2itv1FKdQIWKqU+NM49oLX+W3hnpVQlcAEwHOgDzFZKVRinHwJOBDYD85VSb2qto0P8BSHHeXzudwDUOButE7GWv2tfXMznYYvtX2etjjj/9JcbuPyoA0KvfX6N1aKodXk54X/npjSXed/FDjp94evM9rNrWvwtV/jJv/5r+piPzf2OXx0zKGGfZLrmtW+2kG/vgMbuN67qz86VUYt/s+hRWc+ZDyX84j733HOdJ02atH/UqFGuzp07+z7//PPCo446qv7HP/7xnpdeeqnLj3/84/1Op1N98cUXJdOnT99477339ujcubNvzZo1K+fPn59/+OGHD286Zn19vZo2bdoBH3zwwbejRo1ynXXWWeX33Xdf2W233bYToHv37t6VK1euuueee8ruueeeni+99FJKkdwbN2609+7dO2TyHDhwoHvbtm12gN27d9sGDhzoMdo9e/bsiVrH//73v3cvLS31LV++fFVDQ4M65JBDhv7oRz+qBli2bFnRokWLlldUVLiPPvroIc8880yXn/3sZ6EI8Ndee62kV69enk8++WSdcT9rvPd54403Vl199dXlH3744bfDhw93TZky5cDgOL///e97H3vssdWvvPLKhl27dlnHjx8/7PTTT6/+xz/+UXbllVfuuOKKK/Y4nU7l9aa340HW/sdorbdprb8xjmuAVUDfBJecAbyotXZprb8H1gGHGj/rtNbfaa3dwItGX0FoU6zZURM6tocFm26NEffxeQrWgfAS9w99vA6fX/Obl5pXT6Q5eGPUYGmvzDUK2yUilU/D6TF348d/2B/kj7YZyRVTB+Tll1/ueuGFF+4FOOecc/YEXUTnnnvu/i+//LKkoaFBzZw5s/TQQw+tKS4u1l9++WXxhRdeuAfgkEMOcVZUVESZt5YsWZLfr18/16hRo1wAl1122e7PP/88FMl+0UUX7QU49NBD63/44YeULQqxLHlNiy0mYvbs2SUvv/xyt6FDh1aOHTt22N69e20rV67MBxg5cmRdZWWl22azcf755+/57LPPIvbSGDduXMNnn31WcsUVV/SdNWtWcbdu3Xzx3ufixYvz+/Xr5xo5cqQraBEKjvPJJ5+UPPDAA72HDh1aedRRRx3kcrnUunXrHIcffnjd/fff3/vWW2/ttXbtWkdxcXFaX9YWyR5SSpUDY4H/AkcCVyulLgEWELDG7CUgaOaFXbaZRpHzQ5P2CTHuMRWYCjBgwICmpwWh1Zn8QKMFxBHmEjrv0a9idU+L/5uzlkE9illf1TrxIJ+v3dXuM1yOsCynv6riJd+xKfWPld6cTQpx8iPrPL7xD4ZmBjNnlSQWkWywfft267x580rWrFlTcPXVV+Pz+ZRSSj/yyCObCwsL9WGHHVbz2muvlbz00ktdgkIlFRdgsj75+fkawGazaa/Xm/Ivpby83BO0rABs3LjR0atXLw9At27dvBs3brQPHDjQs3HjRnvXrl2jTBVaa3X//fdvOueccyL8uW+//XanpuKn6etRo0a5vvnmm5Wvvvpq6a233tp39uzZ1WeffXZcn208MaW1ZubMmetGjx4dEQQ2btw458SJE+tef/310lNOOaXi4Ycf3nD66afXxBwkBlm3TSqlioFXgeu01tXAI8AgYAywDbg/2DXG5TpBe2SD1o9rrcdrrceXlZWZMndByJR1O2siYkY+Wxv5ZJ6NJeX7qjpay9jx03+b74LJNZ53/IW/2p8AwJOstC3gytCKUoCTK61v0JvdyTuH8VvbCwC87JuU0X3bMzNmzOhy9tln7966deuyLVu2LNu+ffvSfv36uT/44INigAsuuGDP008/3X3+/Pmdzj777GqAI444ovbFF1/sArBw4cL8NWvWROXvjxkzxrllyxbH8uXL8wCeeeaZbhMnTkx5AY7HwIEDPUVFRf45c+YU+f1+nnvuuW5nnHHGPoCTTjpp32OPPdYN4LHHHut28sknRwmKE088cf8jjzxS5nK5FMDSpUvzqqurLRBwD61evdrh8/mYOXNm16bz3bBhg71Tp07+K6+8cs911123Y/HixYXx3ueYMWOcmzdvdqxYsSIPAoHOwXGOPfbY6vvvv79nMOvpiy++KABYuXKlY9iwYa4//OEPOydPnrxv8eLFadVFyKqlRSllJyBYntNavwagtd4Rdv4J4G3j5Wagf9jl/YBg5GC8dkHISSY/MBe/hgsPDVj9PlgRuTFgNrTFA7PXMKCruaECQmzW7Uxu0Wq6V1KqrMi7HIvSDLZs5TeeK1O+zokDgM/9IzK6b3vmlVde6XbzzTdHpHGdccYZe2fMmNH15JNPrj3rrLOqf/WrXx1wwgkn7AtaR2666aaq888/v7yioqJyxIgR9QcddFBDly5dIrJrCgsL9aOPPrrhvPPOG+Tz+Rg9enT9jTfemNx3GEbfvn1H1tbWWj0ej3r//fc7v/vuu2sOPvhg58MPP7zx8ssvP8DpdKpjjz22+rzzztsP8Kc//WnbWWedNWjgwIHd+/Tp437jjTfWNx3z+uuv37Vhw4a8kSNHDtNaq65du3refffd9QBjxoypveGGG/qtXr26YMKECTUXX3xxhOhZuHBhwe9+97t+FosFm82mH3744Y3x3mdBQYH+xz/+sXHKlCmDu3bt6p0wYULtqlWrCgDuueeerVOnTh0wdOjQSq216tevn+vjjz9eN2PGjK6vvPJKN5vNpsvKyjx33313Wuu5SsUElgkqYDOaDuzRWl8X1t5ba73NOL4emKC1vkApNRx4nkAMSx9gDjCEwEPpGuB4YAswH7hIa70i3r3Hjx+vFyyIvxGbIGSb8lveAeDrW4/n9W+2cPd7kYG03/zxRMb9+cNYl3YIDlbfslb3o5qi1p5KBPm40ChcOChjH/1UFQrNXfYnGWYJWM7Knc/TpdDOotsmJxwr+B1oygFqGy847mSZ/wB+6bmBcLvbILWFOXk3ATDbN5Y/eS/hB90z4vphvUtYtS16p+pfWt/mVvvzXNzjNWZceXw6bzuEUmqh1np8RhcnYMmSJRtGjx5tbhpXlvF6vbjdblVYWKhXrFiRN3ny5Ir169cvD4qatsjbb7/d6f777+/58ccfr2vtuSRiyZIl3UePHl0e61w2LS1HAhcDy5RSwejA3wMXKqXGEHjY3ABMA9Bar1BKvQysJJB5dJXW2geglLoaeB+wAk8mEiyCkEscetecmO3zvkvP9N+e+Ln1PW6zzwDgR647WaYPTHJFS6C5xvo6v7HPZLH/QK7x/JpZRXdQ6N0b1bML1UBmFYvL2MvHeTcA0Mu6l88t1/Jnz095338oAFfb3qBe51GoXJxgXcQJ1kWUO5+PGKN/l4KYoiWfQAr9JRMPymhuQiQ1NTWWiRMnHuTxeJTWmgceeGBjWxYs7YWsiRat9efEdt2/m+Cau4C7YrS/m+g6QWhrpLuHTlunE/Usy/8FAD7d+Gfhrbw/MMz5JA1kVqDUig8fzS8IerRlKb+xzwRgjOU7zrF+RqF3L8v95YywbIjo21XVsCdD0TLK8l3o+FPfKI6xLuUxx9+50n0Nf7JPp0zt53HvaUy1NVpperGbAWonC/RB+LHErRicpzx4tQVtkd1ZzKBLly7+5cuXR2/M1YaZMmVKzZQpU5odc9OadMAiAYIgtDTDwxZ+q4p8WD3dmln21FTrW6zPv5jzrJ8A4CB6n6FkDFWb2JB/Ec84/grA275AYuK1ttcAmJZ3D/d5zgfgK18lAKXUZTRfgALDGnKC614u89zM/Z5zAXjY8SBlaj9zfGN50HsWH/gODl0zL//XvJz3Z060LORu2xMMcK+NOXYeHlzY00qNbUH8fr8/Jycm5BbG9yRuFLuIFkHoYAxWm9mQfxHTrG8x1foW8/Ku4lzrpyj8dKYGM8OEi2jgi7xfc5ft3xHtc30jQ8e32F5goEp/1+Lf2wPZMvfZH+dS6/usyb+Usy2pFdYLcon1g9Dx5e4bWOBvdK24LQV8cespPOQ7kx+57uRB31kAdFaZp5WXqoDg2a+L0Fj4h+9sznXdRpUu4S+eC7nccxM+eyemem5gqPMpXvYewyrLEABOsCzkQtvHXLLjrzHHzsMTCsbNQZZXVVWVinAREuH3+1VVVVUpsDxeH7EjCkIHYZDawi22F1mpA7sU/85Y9AH+Zn+MK6xvMsgSSLAY7HwGrwl/Hn5hfZe+ajcoWO/vzaO+HzFcbeAu70/p4dnLF/nX0kXV8n/2hzjT/eeUxy3AGfH6T/bpgfvZ3uM199Epj7NFdwfAo63M8QesGwPUTn5um8WHB9zMaUa/ZfpABhIQVs2xtJQSEDzhAcgL9FAOcT0a1ddJHjd7pzH9pwdT8cJQBlkCSRYOFXt7mKClJRfxer2/2L59+7+2b98+AnlYFuLjB5Z7vd5fxOsgokUQOgAHqq2hrJQTWRhq36670EsFgk2DgiXQfxtrdH+ay09tjRlS13iuZoU+gFeM11so4+fuG3nS8TfGWKKyNgGYbJlPPm5m+Q/FjZ1fWt9mlR7Io/YHYvavtKRUJT3ETfaXARjharQE3eG9hP/4juDwbpNDogUC1hEIxKV8wnFp3SdIqarDqe240rGIWKzsopQDVeD3Ex6zMrxPSWg/qHzlxqXtWakB1FwOPvjgncDprT0Poe0jilcQ2jkD1XY+cNwc1X66688c5nqIz32BLVWqdWONp+cdd2El9hP9MZYlFBOoaG7DiyW++5kS6tmlS3jUOyVk4QnnI/849gXFgIoWLo87HuBBx0PcaXuSP9pmcKv9eZ513E2xClhafu6+kYe8kWvhzbYX484nHk1FxBIdXVV2v2EdycedsQOtlLrQOKmiAJe209lwLflUo2gp69RYGT5gaclZ95AgmIKIFkEwme9MKKVvx8tnjmv5u/2fcfvk4+IIy3K6sT/hWBdYA3sUXeD+A0e5/o+jXH+nwjmdpTqw4d9PPbdS7nyeUa5/M8L5L/bqYrqrav5mf5Sm8S391Q6mO/7KDMc9AEy3/5WXHHfEvK8VH3nKyzPeydzjvQgd58/NjZ5fAXCsJf6+SefbPuVy23tR7d/4h3Cf9wLKnc/zhu8IAK60vZng00idpvGsGgsb/D3JV+6k19a7Y28Cd0gvS8hiEw8dQxI95Ts5dFxna9zUtzivUcDk4caJPacr+AtCcxHRIggm8/qiLc0e41Lr+/S3VHGm9Uuut82M2Wd1/s943vEXXnTcGXec7uznCttbbNQ9meevZLMuY7PugTtO7EMthfzS/RsAzrJ+wemWLyPOD1OBAmtjLev4k+0pjrSu4BDLmphjFRLYcqSexPvEBSu4XmyLLLYXtObE46fu37GP0N503OX5CdCYAZSMQiMu5nlvansJATSQRyGupC6YVdtiZ5XmeavTt7QoeMp3CqOcj7OEIdh0o2i668yR3HLKUI4Y1E0sLUKHQESLIJiMGSmnf7A/FzoOpt+GE+66GWKJL5JOsQb2BPoijdLui/SQ0HGlZVPEuaAQAbjUlriib1AU1CepwRLMeClpEuA6zhKd2nuW60/c7bmQ7/09+cI/POJcFV1Y5+9Dqjs7TbAESnB85B+XUn+ABhwU4EreMQ4F7j3s0SVpXaOM91NNMXUUYPM33r+00M6vjglYzIIxLYLQnhHRIggm0xzJcrhlBfPzAu4Sn1Zs1YH9xyaoyBpXN9leinh9pGVZzPGCrojpvsQl58PxYeUmz1QA1uk+EeeKVGTWzgZ/T3xaoWLEtRQqw9KiE1taQDHde2KUuBlqWHXu8lwEwEzf0SzSQ3jM9yOOdT8Q093kxEEeyd03ECh4B/Cd7h1nVtHU6zwKVKaiRVPi3Mpm3bih69EV6W3u6sKBzR/7/eVy9pAgmIWIFkEwmXgVS5NxsuVrXnDcRZkKZIP80nMDX/kDBc3OsH4e0fdXtsA+o//1DwVgsIq951gwI6hWp7WRKl8b4/p05J+IIhoA+MFfxmbdnf/4j8CqNN1jxNUErTINSdxDALUU0FnV8b7jZp6w/w2AM61fUq0LWOwfDDQWd0uEE0eonH0ygsG88T6bWL/GoHsoOZFxKeMHdqGUOux+J1t1YzXdKycNCh1/eP3RfHZzYleVEwd2f+z7i3tI6AiIaBEEk8nUOxQMmA3ykX8c9xvVWC+yfRzrEi52/w6AYkNMNCUoJvbSiWnHpL7HT70OWD2aWlaKlAu/Vkx2/5WjXX9nvy4GoEuMgmsFIfdQctGyS5cCcJBlMydaA1scdFXVuHAwXx/EIc6HeNWfvP6KS9vJU6lVxg0KsLo0thBoIC9j91BQ7NTSKJLCvyqDyorpH2OX7vDvkxs7Nh19f63F0iJ0DES0CILJWDIULTt0F6p1Ib/1/JK/ei4AYGvYHjd5uJlkWcS/7fcB8JD39FBA7U32l0PulHA0sEuX4LM46FOaurUluJAXhhVx68VuJlkWU08eDeTjx8J3upfRL3ohDbqH6nRyUfBuk+DZkyxfU0od//EdASiq6BL7wiakZ2kJiJZ4okrFcBA16DwKUsgeikXQreSME3eSithN6B5SnkCdFskeEtoxUlxOEEwm00DcCstmVuqBvJG+sYEAACAASURBVOQLdxE0jjUv7+oIi8anvtEAvOM7lNOsX3O85RtW+wZEjFmg3DhxcNbYvmidenWRBsPNEBQeV1tf50b7K1H9ghaZQuWMqv6fjntoD5HBqY85/g5AdZL04Ka4sKcsWgaqHbi1NW4qdizqyYsQcqmiVOMuzM6wzyP8uxI8TvRriuce0mjycYulRWj3iKVFEEwm0yfd3mp3qKx8OHd7LgQiXTDv+8bztR4GwFWea/FpFbN+SB5unNqRdnBwcCG/zvYaT9j/FlOwQKNFpijGQl7YxD1077mj4t7Pg42bPFOjgnariXaXJCIQiJuae6iP2h23JH6i8TN1DwVFS0OacSfhv7uApSVxTEssC5EgtBdEtAiCyWSyaPRmN73UXmpjuFI+8o8NHf/B8zNu8kxlmuc3EXeM5xYpwB1aJAsc1rTnBYRiTIJ4dOM4QStKrIW8afbQuAGdE97nFd8kKl1PcY37qlDbnLD3ngpObU+p+BsEdoX+xLBWpUq9zsOhfNh0YmEUy1oSnJdTpxksG/Z1cmHHii+6WrHW5CkPTrG0CO0ccQ8Jgsks2rQ37WuON4TBp/7oRXSt7sfbvglMsf6Xz/wj2WjEkYQTT7Tk46aBvIysP5t1d/qpXaHXE5z/pJrCCHdPXZyAXWgUMsnqtDTlTf+RvOk8Mv0JE7BEpOoeKsTFVqItW0FifWZ7jWJ2XahOe24FcdxlU0b15u2l22JdEkUwOygPD/U0isegiHKlK4gEoY0hokUQTOaDlTvSvqa32o1HW/nYPybm+as913J1gof7BmIHiBYoVyilN10L0Lmu2+miavlW96cztVFxJ9AoSGLFeQSFTCoxLWbhSsM9VKhcNKS5yAdFmiPJPZoKHoUi37jG2cQ99Pcfj+Gv58R3nYUTvDYfd4QYtBtVcl3Ym1coSBByHBEtgpAD9FJ72UnntIJCw3Hp2AGoBbjZRWlGY26nG9uNmiKxBAs0xqvEyh4qwIVT2/G3oBfaiZ185SEQFZx49S7ARUOCwnexrnaGLB2JrTmLNu2Lvp8KWloaRYtSYLNasFkbP6OmnqVwsekJEy3h2I00aAnEFdo7EtMiCDnAJMviUK2STGggj/wYwiHPcA9B7I34mosPKy5tpyhGldhCXDHTiXuWZM/yEnSPpGJtCcwvgesqhn/IGUc0NOXOd1ZFtQWvSdeFEz6NkGhqUovGbriH0o6XEYQ2hogWQWhl7HjppmqaJSniBuKqYPZQ9nwGdXHSgAtVU1GQfb9FMBA1mSVE4adAudN2XQUtGflxCtjNWr6N8lveiXkuVvaQLc2iPq44oskR7h4ShHaMuIcEoZU5xLIagDd8R2U8RoN2hLJ1winAFVoksyVc6smPe+/k+w6ZS+Oi7kkYKltgLPrpzi9oyXDo2KLowTnr4l7bWKfFwaCyIs4b358x/RNnVEXdP657SESL0DEQ0SIIJtHg9rFkc3QsQzKed/wFIDqNNQ2cOOhKTVR7PoHictmsklqvY1tainDGdA9l0+oTrDabr9zRwSFh9Fc7gfTrwLhStOTEokC58CkrXmx0LXKEdmeOIsG83eHuobB+4ZYWicMV2jPiHhIEk5j27EIueHxe2td95hsBwHTfSRnfO5Z7SOEnX3mislXCeeeazK07QerJj1lcrkC5aEihhL+ZOEktpiW4w3P4jsvpjZ+maFEB64/Xkv7n0bS4HMS3tEhMi9DeEdEiCCbx+dqqhOe7s5+e7MGBB4XfaNWUqMAC6m2G4dOpHVFF1YILd6IMmeF9Mg/+DVKv80KZMeHEC8TNptUnFHOSRFQEd3hOtC9SrGk2WlpSS6sOpwBXRqIlnHiiyYG4h4SOgbiHBKGFWJB/RcTr41x/46O8G00ZO1Z5+eDrRJYWM6gjnz7sjmoviCdasjiXVC0hQctQ+I7LqdCYnZS+eyhPufFaM7C0hKk8t7KDjhZltpB7yJHx3leC0BYQS4sgtBJjVWPQ5np/72aNFUh5jnz6L4iRrZIN6smPW1yuvqXdQ4aoiJfdE6TI2OE5oaUlxtoftGQkKy4XNRaB34fX0rzAZFeclGfJHhI6ClkTLUqp/kqpj5VSq5RSK5RS1xrtXZVSHyql1hr/djHalVLqQaXUOqXUUqXUuLCxLjX6r1VKXZqtOQtCc/AnCKDsw66otnOtc0PHV3mubda9naHdjRsn0bjXTWZl/FOlXufFzB6K7x7K3mRSdQ9lamlJVqdl5bb4OUv5uFNyDyWqp5NKTIvYWYT2TDYtLV7gBq31MOAw4CqlVCVwCzBHaz0EmGO8BjgFGGL8TAUegYDIAW4HJgCHArcHhY4gtBUus70f1Xa4dWXoeLeOXXE2VZw6D4vSEbEWBaEU2+w+fQcsLfHSrVs25Tn1QNyApSW8joy1Sc2UWFlObmz4tYqb8pyIAtwoR+B+hx3YLeXrYhWXS1SnxfwSgoKQO2RNtGitt2mtvzGOa4BVQF/gDGC60W06cKZxfAbwjA4wD+islOoNnAR8qLXeo7XeC3wInJyteQuC2fRiN1NtsQuOQWDn5irSq9fRlFixHMEKuQHhkN3icgHLReNyaceLXfki3C8tEWqRqqVloGUHFHTBExbW99/fHx/RJ/Z8FS7sGcW05Cs39vwi5t50LNedUBG3X1OxFP7KHScQWOq0CB2FFolpUUqVA2OB/wI9tdbbICBsgB5Gt77AD2GXbTba4rU3vcdUpdQCpdSCqqrEWRyC0JK8mvc/oeMtuhs1utElcZNnKs/6Tmz2PYJxKwXhokW1TBpsg86PsvIEY1xa3NISimlJLCpKqYXS/hFt3YtTm6sLe9oxLRAQkX5rAQO6FUZZdcJp6h4Kf+VXVrzKFvX+7FKnReggZD17SClVDLwKXKe1rk7gz451It6uZ1EWUK3148DjAOPHjxcLqdBiPPrpeu55b3Xc8x/6DuYy2wfc7PklL/uOBQKLer2JFpDwxfriCQOZMW9jiwXi1hnCpAhnKOYi6C6KFdOSTVwpuodKVR3k98zoHvG2TEhGPm78mWQPNTn2quigawdufFrhxZr2+ILQlsiqpUUpZScgWJ7TWr9mNO8w3D4Y/+402jcD4Y8+/YCtCdoFISf412ffJTyvUezXhSHBAsFYCvOeiYMWjQLcTB7ek5OH9wqlPGfb2hGMCylUjRlEweNYZfJzoU5LGfuhsGtm99D2pJacpigV2AfKb0shELfJI1fTz8trcUS5p+zabbgIVYu44QShtchm9pAC/g2s0lr/b9ipN4FgBtClwH/C2i8xsogOA/Yb7qP3gclKqS5GAO5ko00QcoTEq0QxDWlnqaSLs8libdE+BlsC2t7VAmX8gYhg3IKQpSV6kc62aPFrRV4SUVGknFCQWTy/E0dGgbiZWlqa4rHkRaV027Vb4lmEDkE23UNHAhcDy5RSi4223wP3AC8rpS4HNgHnGefeBU4F1gH1wM8AtNZ7lFJ/BuYb/e7QWu/J4rwFwVQ6qzqqdXp73KSLM2hpMVKPT9z3EmfZAsbNhizHtASFSXgp/9ZyDzUGyiZ2D+XjBntmv5NMA3ELcFNra754DbiHorOHXFl2AwpCLpA10aK1/pz4j6DHN23QWmvgqjhjPQk8ad7sBME8klkOuqiaZqc0J8PZpFKrQzcKiGqKshqcGcwQKlTOULRZ0D0UawuBbG6YCAFRkdg9pAOWIHtmAsKJIyoQ9+FP1tG9KL5AU9pPnvJkaGmJ/Lxiu4c8uIzNIrP9+QpCayJl/AUhy3Sinp30yeo9nGEBsFqDWwUWx4OcT0ek9WaDoDUl3D2UyNKS7ZiLZIGydnzYlD+ppSXeNF3aTkmT8e+d9W3CsYLuJL81fctT08/Lo/KiLEkOXFnfrkEQcgEp4y8IzSTZGlyi6qnJsntolw5sfNhPBdL9z9rzL6AxMDWrMS3BQNxw95BqLfdQQFQ0LXMfTkFwnhm6h2JZWpLh0IHPw99M95BSyohpiWFpkZgWoQMgokUQmkkyQdCJeqrJsmihBLe20lnVUVgTyGYK7BqdfVdB0D1UpGIE4oYXl8v6TAIks7SEatlk6B7KJKYluAuzz4RAXK8lXkyLiBah/SOiRRCyiAMPxcqZ9UBcUNRSQAl1jH9rMgDbDr89y/cM0BByDzVaWoJBuS29yzOkIFqC4iqZeyjORF1Ex5QkIy9oaUlBtDQtMtV0Gl7liFkRNxjTIgjtGREtgpBFgu6azbos6/eq0YUcbFkTeu3sPDjr94TYMS1BYdAacRbJsodC83QkFi1N66WExtcZWFqaEdPSFE8MS4sdT+izljotQntGRIsgNJN6ty/uuQEqUDtxk+4Rt49ZbKMbwyzGjhdnPU5tr8Oyfk8AH1ac2h5ZXA4X9ToPHeNPTDZ3eYZAJlWi4m/5zXQPOYm2dCQjZGlJobhcMmLFtORpl7iHhA6BiBZBaCY1Tm/cc/1DoiWzkvHp8KpvYuOL0T+OeORWqKz6ZepDmyYGKMQVKu8fpKX21nDhiCpzH05z3UPODNxDjjTcQ9HzaJLyrKLdXxKIK3QURLQIQhYZoHbSoB1UUZr1ey30B3YO3t332Ngdsqga6skPZQxBQBg0rdESdLdk23vhwk4n6uOebwzEzbC4nLbjwAt+f8rX5OsGAHy2oozuGY7HEm3pceAOq9MiCO0XES2CkEX6qSojniX7S8l3ug+/dl/N2kPujDqX7TiHOp0fFYgbq4R/YDLZncsEyyr6W6qYaFka83wopqUZKc8AeJ2JO4aRbxT786YgWnSTYJqYGyYqD+Eq1C4VcYUOgogWQWgmlgSLcHe1nyptvpUlngh5y38E7sLsu6Ka0lvtYYJlVeh1Aa4YGzW2jIOoTO0HYIxaF/N8fsg9lHnKM5CWaMkLWlrsZlhaAp9ruLUlooy/mFqEdoyIFkFoJokCS7tRzR6yW8I/ZbK4mJWoerqqWn5q/ZBe7KZM7adGR4qClnIPBfFhjdmeqnso3u81E0tLgT/grvLb0rfuxNrlGRq3bEBr8pA6LULHQESLIDSTRItwtiwt8dJxoXVSXt/yBTKV7rQ/xTW21+isatmuu7b8RIBHvT8C4Gb7SzHPp5ryHI9QPRRPQ8rXBN1DvgxES9PftUcFLC2hYGOfO3JegtCOEdEiCM3EEkcl5OGmRDWESuy3FLEEzTEVZVn1znSjOnR8ke1jeqs97KM4ezdMwD3eC0PH49XqqPPnWz8OHGSYftxoaXEl7hhGvm6gTuehM1CU0ZYWQ7QE054Ni49YWoSOgIgWQWgG+xs8eOJkkQQX8t054B46flh241xGWQJbB/zgbyyit0/Hjt/Idp0WgCpjV+2ZeXdEnTvAsiM4kYzGboxpScfS0hA/MLkJ0RVxI+fpUQHRFEp79rZeIT9BaGlEtAhCMxj9pw/iumq6GwGhLW1paQ2men5Dnc7jOPf9NOjA4rm7SZp3S9VpAdimu4WO88Mq9d5p+zcAn/pGZTx2JpaWAn99VIxPJigVIxDXcFOJpUXoCIhoEYQs0U0FLC05I1qyaOD40j+C4a6n8GDjGs/V3Oc5nzd9h0f0MSMQ96aTDkqp3yPe00PHwarEoPmpbQ4AT/lOyngOmcS0FOgGamm+aIHwmJZIS0tjnRZJHxLaLyJaBCFLnGH9AgjswJwTtJCp40P/eB7ynUlDHHdIc7xDQ3t1Sqnfe/4JnO/6I0BoP6agZWK/LuQT/5iM55CZpaWOugwtLfGyhySmReiIiGgRhCxxpvVLAHa0UhZNrqFb1EEE63RfAGwE9oY63/oJAHP842iOvSfTmJZaChJmfQVJ1sdrWFq65RmxVE1iWmTDRKE9I6JFELLEQv8QanU+HmytPZWcoNE91DKraoOxiBcYMS0j1fcA7NBdmjVuxjEtZrmHjJiWw/obgc6GeHJpCcQV2j8iWgQhSxxsWUuxSr0AmVnEfVDPkSfwlrIEOEOiJeBGmWRdAsDD3jOaNW5mMS311OnUsoeSfT5uQ7TYjU0YQzEt4h4SOgDyCCgIWcBC6pvpCamTinsl1BcLTm2nwIj96KH2AaRs8YgnHlyZ1Gnx16cciNv0PUbOQ+FVAXFi07FjWnJEmwpCVhDRIghZYLJlAQBzfSNb/N5xF62WDSnJCRrIo4Cm1q7UlvV4AindmBYHHhx4qNEFpvwKgu6hzu4d8Oce4AuIp2odqLbbAX/NQgdC3EOCkAUutH4EwBf+Ea08k9whHStJPNJ1LTXgoAB3yPL1gOccAA7onvnGhSHR4knN9VdEQNzUpWhpGdk3cYp8MBC3X/3KkGBh4g1soSzBVYLQPhDRIghZYL4/UE/kKd/JLX7vXI1paensIYAGnUeBcoWCcRtwcM64frx59ZFJr40vkBRO7ClvmFhk7CpdH7XrdWxmXH5owvM+ZcOPotRjVPa98r9w/G0pjS0IbR0RLYKQBYqUE7e24pbgyChaoox/ECcO8nHTmVoA9tKJvl0K6JTfvN+LC0fKoiUomFINxO1cGJkFFJVtpRQuHHR3bQ68Lu6R0riC0B4Q0SIIWaAIp2kVUNMlVwMxzXAPpUsgpsVFYdDaofNN+XzcaYiWIiOmJtW9h5oSS+NVE+beyu8c2T+juwhC20BEiyBkgWLVkHEF1GxxYDPiOMykJRfVBu2gQLlDwqGOPFNSrp04Uo5pKTTS3lO1tCRDa/iXOjfwoqQfWOTPuNBxSPnbrpQ6Sin1M+O4TCl1QJL+Tyqldiqlloe1/Y9SaotSarHxc2rYud8ppdYppb5VSp0U1n6y0bZOKXVLem9PEFqHYszbayZd4hk0xpfnRmXelqzY6iSPAtwUhFlaUiVREbx03EOFNMa06Gaam4Kf3auWydw94h2YNrdZ4wlCWyMl0aKUuh34LfA7o8kOPJvksqeBWFGID2itxxg/7xrjVwIXAMONax5WSlmVUlbgIeAUoBK40OgrCDlNwD1kzpN1e6F13EMO8nGFuWhSC4ZNhiuNQNzCZrqH4lFv6wxF3ZJ3FIR2RKqWlrOA04E6AK31ViDhzmVa67nAnhTHPwN4UWvt0lp/D6wDDjV+1mmtv9Nau4EXjb6CkNMUqwZqc8w91B5IV/gEsofcpguHdCwtRSa7h5LRkoHOgtDSpCpa3Dpg19QASqnmOMevVkotNdxHwU1A+gI/hPXZbLTFa49CKTVVKbVAKbWgqqqqGdMThOZTTEPKdTnMJleXLDPX0oqexSn1C9RpcYWEQ73OM2XvIzf21GNaDMHUYJKVB2i2m0kQ2iqpipaXlVKPAZ2VUr8EZgNPZHC/R4BBwBhgG3C/0R7rr4hO0B7dqPXjWuvxWuvxZWVSZEloXYqUk5pWsrR0hOWsX5fClPo1GDEtXakBYD/mBCNnGtNiJvFEoAgaoT2TUhl/rfXflFInAtXAQcBtWusP072Z1npH8Fgp9QTwtvFyM9A/rGs/YKtxHK9dEHKWgKVFYlrMJu2KuDqPPOWhRNXh1HYayDfF4hMQLTUp9S1SLtzKgQ+rSYJSRInQcUkqWoxg2Pe11icAaQuVJmP11lpvM16eBQQzi94EnldK/S/QBxgCfE3A0jLEyFTaQiBY96LmzEEQzGLj7rqY7Qo/hbhEtDShNQwAdYZ1o4/anbalI5G4cal0LC1OnMocq1uuuv4EoaVIKlq01j6lVL1SqlRrvT/VgZVSLwCTgO5Kqc3A7cAkpdQYAo8KG4Bpxj1WKKVeBlYCXuAqrbXPGOdq4H3ACjyptV6RxvsThKzh9cdehfNxY1E6rfTajkRLxonu1oF9fAarLaZm77h0GjEtyoXXGhAthXZr2vdK9/OSQFyhPZPqLs9OYJlS6kOMDCIArfU18S7QWl8Yo/nfCfrfBdwVo/1d4N0U5ykIrU4whkEsLa3PMMtG498fWOuPGcOfEeExLV+u25WwbyFOOpd25t6TRnHoAblRK0cQ2iqpipZ3jB9BEJIQrIDaoM0NvBTS50nvKUyzBf50NRDY08ecMv6NdVoe/Ghtwr7FNGDJK+L88f0T9ksHiWoROiqpBuJOV0o5gAqj6VuttSd70xKE3CdejEaRWFpi0hq7PFfRuC9PuinHicSNU+WBzw1+X9JxipQTHOZbWMQJJHREUhItSqlJwHQCcSgK6K+UutQoICcIQhjHW74BwEHr6Prepe1XLKUbzOsPq+pQb1i+zMkeMnaJ9rqS9i3ECY7U6sqkgmQ0Cx2ZVN1D9wOTtdbfAiilKoAXgIOzNTFByHXiLX6X2D4AGiuhtjQVPRMWq241zCjq1jhW+gSzh04e0avZ93cZrqZUMogKcYGj5TarlDhcoT2TanE5e1CwAGit10DwUUMQhHB6qn0AvOE7spVnIgChIn+1OlCQbnCP5os6d8jS4kwqxgqVCxypFcNLhggSoaOTqmhZoJT6t1JqkvHzBLAwmxMThFwnnpn+c99wABo6eEzL4B7muUSako6HZKE/EIr3gzavUrYzaGnxNCTtW4AL7OZaWhK5iMR9JLRnUhUtVwArgGuAawnUU/lVtiYlCG2ZHXTlB79sJdGvS2RBNTMCcTOxNATvWmviXlD1GJaT+t1J7x5wD2VuaYlnyZF6LEJHJNWYFhvwf1rr/4VQlVzJ5xSEGBTilMwh4sedmBnbkg5mVsStCYoWTz0k+F3n4cGiNNjNcQ8JQkcnVUvLHIh4TCkgsGmiIHRY4i1qRThN3xxPyBxtiCQz6+aE3ENJsoeCOzxLIK4gmEOqoiVfa10bfGEcy6OD0KGJW6dFOamTEv45UwAtKFpCQsMEXCnGtASrI5tpaZGYFaEjk6poqVNKjQu+UEqNB5JHoAlCByTgHjIvfkJoHsE1Ps/EujluZVhtklhaCpRxvjkxLWGWk9ZyrQlCrpBqTMt1wCtKqa0E/gb0AX6ctVkJQhumCGdod+HmMLJvKcu2pLxHaYcgEytDsBCcJU3bTyKB4NTBlOeGhO6YRktL5u6hWO9Zi7lF6KAktLQopQ5RSvXSWs8HhgIvEdiFeRbwfQvMTxDaHIXKldEOz+9eM5FPb5oUej1lVG8TZ5U5L009rLWnEEU69oZZvkMBWKEHpnWPRNlOje4hZ0IhVWiCpSVdxBYjtGeSuYceA9zG8eHA74GHgL3A41mclyC0WYoyzB6q7FPCwG7ZDdj86nfH8eUtx6V1zYQDu9G92Lx4kJbmLf8RHOR8mvXavF2enWHF5RJRYIKlRRCERpK5h6xa6z3G8Y+Bx7XWrwKvKqUWZ3dqgtD2sOCnQLkzsrS0BL1LWy7WJpsejHSHdmUQhJvIPeROUbSE3EMmxbQIQkcnmaXFqpQKCpvjgY/CzqUaDyMIHYYiI8XVjJiWXIpaMEOAmDFGrizgfmUFix08SWJagvtPmZk9lOSbkUvfG0Ewm2TC4wXgU6XULgLZQp8BKKUGAxIhKAhNKAyJFskeikeuCI9mYy9Inj0UsrSY4x5qN5+dIGRIQtGitb5LKTUH6A18oBtD1i3Ar7M9OUFoawR3ds5V91BL0tYX2ETzVwC2fPC2fJ0WSGxNaeMfuyAkJGmdFq31PK3161rrurC2NVrrb7I7NUFoexSa6B5qDc4Z16+1p9B2sOWD15VQ3BQoF36tAlYZk2nrolAQMiHV4nKCIKRAkfFkXW/C3kOtsSYd0F0KXaeMPR88DYlTnnHRgKNZCkO0iSA0IqJFEEwkGHjZUcv4L//TSaHjbGQP5VRNNVt+StlDzd2HKpfesiC0NiJaBCFjopeTYmN3i7a6y7OKYxFIdeHMt7XMn5ScsD6kIFoKlMvUjRoFoaMjokUQTCRYATUblpZTR/YyfUwhM5RShnvImbSMvxmuwnCSWZsk1kVoz4hoEYSMiV4dgnVazF6oAM4f39/0MVMl1b1u4llq2hta6xSzh5w0NNM9pGIdi89I6KCIaBEEEykMiZb2VVwuVRJJFjPfT058Nkb2UCIKlIv6LLmHZMdnoSMiokUQMmDVtmpO+N9Po9qLlBOXtuMNK4EUHpwqNC8eJVcMORoCacyehoTiwYxA3HTJqWBlQTCZrIkWpdSTSqmdSqnlYW1dlVIfKqXWGv92MdqVUupBpdQ6pdRSpdS4sGsuNfqvVUpdmq35CkI6vLd8e8z2QpxRNVqK8zrOjhe5IirMIKmry5YHXlfCsvoFuJrtHhIEoZFsWlqeBk5u0nYLMEdrPQSYY7wGOAUYYvxMBR6BgMgBbgcmAIcCtweFjiC0FEs372NHdeIskSDFymlaPEvTJbMtxIukOsdXrziC26ZUZnk2WcZWkDymxQT3ULq/9jbwNRGEjMmaaNFazwX2NGk+A5huHE8Hzgxrf0YHmAd0Vkr1Bk4CPtRa79Fa7wU+JFoICUJWOf2fX3D0vR9HtMVbFwpxdtgaLelw8MAuHDWke9rX5ZTrw8geSkRWsofIkZgeQWgFWjqmpafWehuA8W8Po70v8ENYv81GW7z2KJRSU5VSC5RSC6qqqkyfuNCxcXn9KfUrwpm1Gi2pZvA0h3hP6bJIRhLae8jnQiX4cArMKC4XNn64JUssKkJHJFcCcWP999MJ2qMbtX5caz1eaz2+rKzM1MkJQqoUKpdYWlqAnFivbYHfsx13zNN2vNiVL2vZQ4LQEWlp0bLDcPtg/LvTaN8MhBeh6AdsTdAuCK1KvKfcIsyLaWnrtHXrTFJhZIgWh46d9lxgpL9b8oqaN4+cUGiCkBu0tGh5EwhmAF0K/Ces/RIji+gwYL/hPnofmKyU6mIE4E422gShVYmX5lpoonuoNRb9eO8rp2JJcgW7YWnRnpinC43NM6edMKJFpjOmf2fjSFSO0H7JWi6mUuoFYBLQXSm1mUAW0D3Ay0qpy4FNwHlG93eBU4F1QD3wMwCt9R6l1J+B+Ua/O7TWTYN7BSFnKFJO6v1t1x3Qu1SsRCljKwCClhZH1OkCFXAbdSoubZHpiEVG6AhkTbRorS+Mc+r4GH01cFWccZ4EnjRxaoKQEa8u3Mw5KxjCzAAAHeBJREFUB/eLe17hp5Q6qmmeO6A1OWNMH/7x0VrWV9VFtKe7IA7sVhjVZmYgcUsYfpK+Z1tAnNq1m5iixbC0YC8wdV5at0xQtiDkIrkSiCsIOc8NrywJHcda0Eqox6587NKpPVnnJdkRuTXqtCilOG5oj6j2dNbIVXeczAfXH53oJhnMrNmXmo89aGmJHYgbEi2OaAGXHrHfdC59FILQUohoEQST6K72A6QsWsaXd6FHp8xcSZ1yuMpugcNKns3a2tPIKhoaA3GJE4hruIewN1e0CIIQRESLIJhEN6oB2EVJytekYzkI7/rWr4/ivnNHpX5xO6ElvSKpZw8lsbSY7B4ShI6MiBZByIBYC1o3FRAtu3XqoiWdRTi8a3n3Is4b3z9uX7MxI4bCTL3REq6RpPMNZQ8lEy3NtbRI/IogBBHRIggZEMtC0ihaWiZbpLX50+nDW3sKrUuyOi3iHhIE0xHRIggmEXQP7aU4pf7pGi9aM/CyS1F0dsylR5SnNUZbCxxNNN9QGX9SsbQ01z0UPZNYXx1JKBI6AiJaBMEkSlQ9tTofH+YEoQ7s1rKp05ccPjDuued+MYHxA9PbYL3dp+UmzR7KjqVFG5IlXjxUTmVYCYLJiGgRhAyIlX5cTAM1mLdAnTyiV5N7pnf9yL7puanuOCN+5dZ+XQr56WHxRU17JGmKeUSdlmgKlBOfVqF+5szJtKEEoU0iokUQTKKTqqdGp+4KyLYhwmbN7L93tubV7uwuERVxoynATQN5ojQEwUREtAiCSXSinlqyl94ab1+gtoYZ7yInBFAySwsuGmJUyk0X0TyC0IiIFkEwiU6qgRotmSLZpCUX8KT3Ugps+fFjWpQLp267+1AJQi4iokUQTKIT9abGtDSlrT1xZ2PbgWy5rqYdc2BmF9ryE1ha3NSTHdHS3mOcBSEeIloEIQNircfFqiG9mJZQHkiK/dvYQpXN7CGz5dDvThmW3v2DXwBbfoKYFnPcQ92appvrJnMQhA6EiBZBMIlOJmcP5RrpSSzzufPM+NlNLU1IkNkTWFqUG6cJlpbOhY2iRXSK0NER0SIIJmDDS6FypWVpSTewVhas3CEk32wFCfceatDNt7QIgtCIiBZByICmgqMIJ0DG2UPTf35os+cUj7PG9s3a2K3JMRVlrXbvkOfLlpcweyhbMS0x59RidxKE1kNEiyCYQCdVD5CWe0ijmTi4OwBjB3RO2j8TQ8uGe07jgR+PyeDKWPc319RzYPciThreM+X+Lb0op/Ru7YksLea4h2KRyFUnBjmhPSOiRRBMoBRDtKSZ8nz3OSP59KZJlOTbszGtnKJpXK7NauGxi8dnPkArEnLV2fKxxwnEzVfZdQ+JOBE6IiJaBCEDmsaXdFY1AOzVqW2WGCTPZk19j6F2skq1q9icOCnPCj+l1GUlMDt3pJsgtDwiWgQhA5quu12oBWAvnVIeI4cMBxHk6LQi1E6LzDEVdWWPXVyuEw04lI9dOr39nxLRvdjRbqoiC0KmiGgRBBPorAKiZV8alpZ0F15ZsCLJtnCxpiJa4tRpKc3g+5CI2b85mg+uPyZhn7vPGskxFWVU9ikx5Z6CkIvYWnsCgtAe6GxYWvZhziIVi9auk5JLKLJbvO7nRx7AOQfHz7pqzB6K7R7qr6oA2E+Krr8kDO6R3IJX2ackq1logpALiGgRhAxo+hBeouqp13l4mvyX6tclexsotjUO7B5YwC87ojyj61vSznTbjypT6xgne6gH+wDYobuYOa0QuepaFIRsI+4hQTCBEuqoaVKj5dfHDeadayZGtP325KEZ36Ol3EPZukuXIgcb7jmNM8Zkp25M/66tIBDj1GkpVg0AbNPdsndv8RYKHRARLYJgAp1UfVS6c/+uhZQWRKYyTxzSvSWn1aFolZgfWwE2vFjwRzSXBFPgMyw2KAhCbES0CEIGNF0gO9FAdQrpreFupXSX2JZKFY7neZCYmkbC9x4CyCfS2lKi6nFpOy4TNkyMe29B6IC0imhRSm1QSi1TSi1WSi0w2roqpT5USq01/u1itCul1INKqXVKqaVKqXGtMWdBCCdWTEu6heVk6WkH2AKiJa+paKEuJRGbLu2qxo0gZEBrWlqO1VqP0VoHS2LeAszRWg8B5hivAU4Bhhg/U4FHWnymgpCETtRnfYfnjr5etYTIO2dcv/QusAUtLZ6I5hJVT3WaIlYQhOTkknvoDGC6cTwdODOs/RkdYB7QWSnVuzUmKAjx6KQaqE5jh+e2SLoxIy3txfjDacOaPcZ9547i2ztPTv2CoGhRkZaWbItYsdIJHZXWEi0a+EAptVApNdVo66m13gZg/NvDaO8L/BB27WajLQKl1FSl1AKl1IKqqqosTl0QoslokUpz5VE55BsY2iv1yr9m0qs0IBKGxyigNnl4L/oY5zPFYlHk2aypX2APuociLS1jLOuaNY9UkGKDQkekteq0HKm13qqU6gF8qJRanaBvrP+ZUX/utdaPA48DjB8/Xh5EhBbDjpcC5c4gpiX213RE3xLGD+wa89w71xzFV+t3pz1Hs3njqiOT9smGxhrRt5S3f30Uw3qXcNlTX5t/g3SxBaxr4YG4x1m+oVTVs8ffOsJOENozrSJatNZbjX93KqVeBw4Fdiilemuttxnun51G981A/7DL+wFbW3TCgpCATqH01kjRkmfLzJD59q8nxmwf1a+UfLuV4X2av5/NxzdOwuf3xzyXilsn356GNcIkghpoRN/A+8+JJBpbHhBpabnW9hr7dSG3eX+WlVvmwtsWhNaixd1DSqkipVSn4DEwGVgOvAlcanS7FPiPcfwmcImRRXQYsD/oRhKE1iLcVdNJBURLeODljZMrmDKqj6n3NFMoHNC9KKXS8LE4Y4y57ytTrpw0qLWnAHbD0mLEtCj8DFWbeMF3HJt1mem3E4eQ0NFpDUtLT+B144++DXheaz1LKTUfeFkpdTmwCTjP6P8ucCqwDqgHsvP4IggZ0o1qINLScvVxQ5Jel0pMwgnDejB71c6k/czELLdOti0hfTq3XuBz6K3ZIuu0dKeaPOVli5YigoKQDVpctGitvwNGx2jfDRwfo10DV7XA1AQhZcLX9XK1HYDd2vzdda+YNLjFRUtbISfcJLbIQNy+ahcAW7NZvh9y5M0LQsuTSynPgtAmKVV1AHynWz8Tf1S/5se7hNO7mdk47Z5g9pDhHhpnWQvA1hawtORQMpkgtBgiWgShmQT3manNgX1mMt1BOZxwt46si0lo4h6y4wVgg+7ZalMShPaMiBZByIDwp9xuqppqXYCP9AJl29pePi0930sOH9ii90uL4EdhD8QxHW1ZBkCJqsOjrTSQl71bt62vjSCYSmvVaRGENk24BaKn2puRO0AWn8TcccYInvlqY+h104+rJD8H/nzlFQNQbVjZSqg39hxqno1qxZ9Owm61YGkyTC4VGBSE1kAsLYLQTLqqavbo9llILHyRTLcC6xUmpCSHV7790ajImKFuxXl8dvOxEW2tsahvdwwMuYc6ZbBxZiyK8mw4bBZs1th/otualU4QzCIHHlUEoW3TlRpWkcOujFbiyMHd2XDPaZTf8k7a12645zQA3v71UUB8MdK/a+tvSui0FNGJBiDc0pJ9xOYidETE0iIIafL20q28OL9xO6xuqprdGVha0nlWHjugc9rjtwTHD+2RvFMzUEqZaj1xZFilOBGF/mqOtgZiWnqrPVTp3PxdCUJ7QESLIKTJ1c8vYvX2GgBseOms6thLbriHzFjfpx59IF2LHCmNd/yw1LJk7Nb4A8296di458wmPwuiZb8vmBauGah2sEH3Mv0egiAEENEiCM2gC7VAZoXlzLAfLL7tRBNGiaRXaT4vTT3MtPH+/uMxvH/d0XHPD+jW+i6e5vCR6yAatIPuVFOoXGzS2bU+CUJHRmJaBKEZdFWBEv57UhQt6QazJqNpoGZrZiSdMiK2heHMsX1beCbxMcvVFP4xN5BHgXIzQO0A4Ics7DkU796C0NEQ0SIIzaCX2gvAzhTjGNLN+ih0BGq/9ClNrXDdM19t5Oxx/dK6RyLSWeN/ftQBpt23pVhz5ynNHqNeB2qyDLFsAeCHLFpagr8OSZcXOioiWgShGQxRmwFYq7NjTRjWu4QHLxzLsQel9vS+bMt+U++fjmWoUy7UTUmTTANza13e0HE9gZiWYSpQU2ZzWM2ef140thmzS4yUbBE6Im3vr4wg5BCD1FaqdAn7UgzEDRcBqT4snz66T8rzMWsdqzEW5U176lPqX96tkKG9zN8wsi3QYFhaxlrWscHfkwYa92uaMir1350gCMmRQFxBSIMv1u2KeN1f7WRzhu4AncM2fq8vtbnZjKygiUOyG8eRq1TVuChSgRotoy3fsUoPaOUZCUL7RiwtgpAGP/nXfyNe91O7WKZbL5bDDOFzYmVPxg3oEtFW3j2Q0VNaYG+8Vwzb0Jlj+rJ+Zy1XHTe4WXN479qJ7G/wcMHj85o1TjIe/sk4U8c75K7ZjFeNhQVX+aXIoCBkExEtwv+3d+dRUpVnHse/T1VXd7M0NNDQsjTK4oIKAjK44CgJLqgZJYZzxGV0JomOmmRiMk6OxtGo2XROJmY9cY86RnGicYvGYKLRaMQQFBXFpUWiiLLa7A3dVc/8cd9uil6gbbq66lb/PufcU/e+dfvW+zRF11Pvcl/phF7Uc2XJ/7JPYiWPN07NWz1aphGdSWFuPmdKq7JEGDCxu6SotCTBZSeN68Sr7mzc0O7pWhqZgzvovpe1onN3tLQUcgudSK6pe0hkFxrSmRYlzuzk0ywp/zxnlDwFwIc+sPsrFvQrT3H3Fw/r8us2z1LZqUwjP7OlM9FvZxU7Zo4t8Ry3tIR/AqUt0lOppUVkF7LHdvRjEz9M/YJjky+x3KsYYdH4llrv3GDL6n7luz+pA44c+8lXmN6dppYWfTq2b8mHG8LejmRueSdW++4sJZHSEylpEdmFH8x7E4Ax9gE3pX5Ija3imoZ/5vb0CQDsbSt513deffjKzxzY7vXGDunbvJ+LKcLJLpoHW6zTabuyZyWR9UsaXX8XpTTQXcsYpjO+07RrkZ5C3UMi7ahvSHPrs+8CcGPqeqpsPWdtv5zb0ieSIUGGRKuEBWBQ39J2r1lakuC6z43PWZ27KtlounNssTW01Azs2E36WsoekNwkkfXXM0OCeso6W61PpHZVtHTE7X9Z1i2vJ1JIlLRIj/bWyo2ccdN8tm5Pt3rugCseB2Av1jI2sYKfNH6WBX5Aq/OOO7BjiwZ2hy5rIQnZSjE1uJwxtaZTt/F/5j8/xZ8umd6qfHtjy/FO3UMtLNKTqXtIerSrH3mN55euZd7rH5HOePMt8Gf+6Jnmc75Y8hgAL2b2ay6bNnYQ5x89hk31jSxYtu4Tveb+4SZsh40atKfVzxkLX2cO3WfArk+MkSPHdG68SXsLOp7ys+f2pDqdpslD0pMpaZEeKZ1xrnv8DZ6rXQvAV+cuAuD1FRv4w5KVLFsb3Ql2jH3A7OQzPJ2ewE8uOY+aQX3YUN9A71SyebHCo8ZWsa0xzeqN2/nDkpVtvt7Q/uWcfXg0s2RiTSULLj+WwRXd053QGf3KU9x/4ZEcsFfH7vRbiM4+fCR3zX+vW17r2tPGM2NcNXNuep53Vm8G4PgctMAZmvIsPZuSFumRxnzzsVZlk+xtKuffyxzSVKU2MMZWMClRy9ZEXyac/QMGDOoDRB/o2fr3TvH90ybwlXteavf1nr9sxk7HuUpYuvLz7NC9u7+V5dhx1e0mfp/Ud2aN75akZfr+g5kzNbo/y6iqPs1Jy01t3P+mKyhlkZ5MSYv0eEaGr5fcx1dKHqTRE6RJso4K/u7VvDfhYkYedxG9Knb/rXnWxGE88vIKDhnRsRWfc2HGuNytMNwdbjh7MtvyNFaks2499x+a978zazx/WPLHnL5eRi0t0oMpaZEeZ9H7dc37/dnED1I3clxyIXMbp3N14znNC9499KVpjKzpeAIyY1w1y649ucvr2xG/OGsy//Hrl/neZ3M3M6k7lCQTzd1uhebu8w7jzJt3LONw+pQarpl1EMnEjsG9e/XvmnvviEjblLR8AhvrG/hofT37Vrffzz9/6VrSGWdaB2/4ta0xzQtL13HFQ4t54KJpDOyzY7psQzrDus3bd3sTstuefZfDRg/koGH9OxZIDzfr588BzszEAr6VupPq5EbSx36fSx8eCRj/dsxoLp6xH71Kk/muaoedOH4oJ45vPf26p8nleI/sgbwV5SVcN3tCm+c98uWjWL+1IWf1WLlhW86uLVLoYpO0mNlM4MdAErjF3a/t6tdo2FLHd797BWU0sIleeDsTPi30KluL3uXs83/XTs9z9s9k708Hfvy9B1qVO0YGC/cFyXr0BOmw7xh/C/cN+fGZUyirGAR9q6HPYCjtu/MNJYrQh+u3csT3n6QvWxhq6xhmaxlmaxhqaxnExvBbcxI4hvM/KWesfcAhiaVQfTCc8lMYPpllR+Y7ksI28+C9+Kc3hnHZiXu+1lCurFhf3y2v87Mz2194cfyI3H15+Kib4hMpVLFIWswsCfwcOA5YDiwws4fd/fWufJ2V6zZyVerOrrxk97tv58OMG5spZxO9qPdUc9O70XpAXybjO/WXlyQTzec1Zq3B017zfdM5JcnETucbhPQKShKQMA+pVth852N3bz7fcBIWJRyE8xIW7TclIUPNWdZGY1TajY+pIB3ObEr63I2N9Gb1Mdcy+OjzIBmL/wZ5V55K8tMzJuW7Gq385qIjm1sjjxpbxbO1a/jR6RMBmPe1ozn++mj6elfcT+fN78zkL++s5Zj9Bu/xtTpjcxv3ExLpSeLy13oqUOvuSwHMbC5wKtClScuIETU8dMR9XPPUahJEH7r9ylNsqG9g3yF9Of6gav781hq2bE9Tu3ozE2sqOXhEfxb+/WOWrNjA1FGVVFWU89grHzFtbBWLltexqT76I+PArEnDWbBsHXVbG9lUH90gasaB1by3dgsrN9Rz5Ngq/vz2Ggb3LaNuawNrNzeE9hNneGUZH9VtZa+KFNPGDKSi1Fjw7hr6lSVY/EEdCZwkGabu3Y8hyc1UNK6lIl3H+x+toi9bqUxuo7K0kf69SzGiwXwJs+abkbnD6o3bcLx5IORevXtRkjQSwPK6rWTc6VeeorJ3i7uDehRf3ZaG5i6V1Ru3kSqJLl6aTLBhW4bSkgSDK0ILliWipMkSoYUqpCxmbNqWYdWm7TRmoE9ZiiH9yml0aMxAaUkJ9Y0ZtjY4pakkm7dnGFbZmwF9Sjl4n2F4v2EkB4yEfsPZUlpFY4NRUV5CKpnAccpK4tPlIx0zeeSOWU53tVg8cr/qCt749ky2pzOUp/b8376sJMmn9i+Mwc73X6imQel54pK0DAfezzpeDuz018nMzgfOBxg5svPLw596wnGcekL7z//LLp5rcvWZnX75gtV+Y3jhqgib9GzlqWSXJCyFJh9T0kXyLS6DHdoaXLJT74a73+TuU9x9yuDB+Wm6FRHJpRevOI7xw/u3uayASE8Ql5aW5UBN1vEIYEWe6iIikhcD+5TyyFeOync1RPImLi0tC4B9zWyUmZUCc4CH81wnERER6UaxaGlx90Yz+zLwe6Ipz7e5+2t5rpaIiIh0o1gkLQDu/hjQesEYERER6RHi0j0kIiIiPZySFhEREYkFJS0iIiISC0paREREJBaUtIiIiEgsWC6Xcs8XM1sN/H0PLlEFrOmi6hQixRdvii/eCjm+vd1dtxSXglWUScueMrO/ufuUfNcjVxRfvCm+eCv2+ERySd1DIiIiEgtKWkRERCQWlLS07aZ8VyDHFF+8Kb54K/b4RHJGY1pEREQkFtTSIiIiIrGgpEVERERiQUlLFjObaWZvmlmtmV2a7/p0lJndZmarzGxxVtlAM3vCzN4OjwNCuZnZT0KMr5jZ5KyfOTec/7aZnZuPWNpiZjVm9pSZLTGz18zsq6G8KGI0s3Iz+6uZvRziuzqUjzKzF0Jd7zWz0lBeFo5rw/P7ZF3rslD+ppmdkJ+IWjOzpJm9ZGa/DcdFExuAmS0zs1fNbJGZ/S2UFcX7U6SguLu2aFxPEngHGA2UAi8DB+a7Xh2s+9HAZGBxVtl/A5eG/UuB68L+ScDvAAMOB14I5QOBpeFxQNgfkO/YQt2GApPDfgXwFnBgscQY6tk37KeAF0K9/w+YE8pvAC4M+xcBN4T9OcC9Yf/A8L4tA0aF93My3/GFun0duBv4bTgumthC/ZYBVS3KiuL9qU1bIW1qadlhKlDr7kvdfTswFzg1z3XqEHd/BljXovhU4I6wfwcwK6v8To/MByrNbChwAvCEu69z94+BJ4CZua/97rn7h+7+YtjfCCwBhlMkMYZ6bgqHqbA58GngvlDeMr6muO8DZpiZhfK57r7N3d8Faone13llZiOAk4FbwrFRJLHtRlG8P0UKiZKWHYYD72cdLw9lcVXt7h9C9KEPDAnl7cUZi/hDd8EkotaIookxdJ8sAlYRfVi9A9S5e2M4JbuuzXGE59cDgyjc+H4EfAPIhONBFE9sTRyYZ2YLzez8UFY070+RQlGS7woUEGujrBjng7cXZ8HHb2Z9gfuBi919Q/QFvO1T2ygr6BjdPQ1MNLNK4AFgXFunhcfYxGdmnwFWuftCM5veVNzGqbGLrYVp7r7CzIYAT5jZG7s4N64xiuSdWlp2WA7UZB2PAFbkqS5dYWVociY8rgrl7cVZ0PGbWYooYfmVu/8mFBdVjADuXgf8iWisQ6WZNX2xyK5rcxzh+f5E3YOFGN804BQzW0bU5fppopaXYoitmbuvCI+riJLOqRTh+1Mk35S07LAA2DfMaiglGgT4cJ7rtCceBppmH5wLPJRVfk6YwXA4sD40Xf8eON7MBoRZDseHsrwLYxpuBZa4+w+zniqKGM1scGhhwcx6AccSjdt5CpgdTmsZX1Pcs4En3d1D+ZwwA2cUsC/w1+6Jom3ufpm7j3D3fYj+Tz3p7mdRBLE1MbM+ZlbRtE/0vlpMkbw/RQpKvkcCF9JGNKr/LaLxBJfnuz6foN73AB8CDUTf1r5ANA7gj8Db4XFgONeAn4cYXwWmZF3n80QDHGuBf813XFn1OoqomfwVYFHYTiqWGIEJwEshvsXAlaF8NNEHcy3wa6AslJeH49rw/Oisa10e4n4TODHfsbWIczo7Zg8VTWwhlpfD9lrT345ieX9q01ZIm27jLyIiIrGg7iERERGJBSUtIiIiEgtKWkRERCQWlLSIiIhILChpERERkVhQ0iICmFk6rNDbtO1ylW8zu8DMzumC111mZlV7ep1wrT5m9kTYfzbr5m0iIkVBf9REIlvdfWJHT3b3G3JZmU46Apgfbky22Xes7SMiUhTU0iKyC6El5Doz+2vYxobyq8zskrD/72b2upm9YmZzQ9lAM3swlM03swmhfJCZzTOzl8zsRrLWmzGzs8NrLDKzG8Miikkzu93MFpvZq2b2tTbqOCYstngXcCawEDgkXGdIy/NFROJKSYtIpFeL7qHTs57b4O5TgZ8RrZvT0qXAJHefAFwQyq4GXgpl3wTuDOXfAp5190lEt3MfCWBm44DTiRbemwikgbOAicBwdz/Y3ccDv2z54u7+TviZhURr3twJfMHdJ3q0Fo6ISFFQ95BIZFfdQ/dkPV7fxvOvAL8ysweBB0PZUcDnANz9ydDC0h84GjgtlD9qZh+H82cAhwILwurVvYgW2HsEGG1mPwUeBebtIoYh7r7WzMYDN+8uYBGRuFFLi8jueTv7TU4mWkvmUGBhGABrbZznLR6zGXBHaB2Z6O77u/tV7v4xcAjRys9fAm5p9YNmN5jZYqIFPxcBM4FH2+pKEhGJMyUtIrt3etbj89lPmFkCqHH3p4BvAJVAX+AZou4dzGw6sMbdN7QoPxEYEC71R2B20xiUMCZm7zCzKOHu9wNXAJNbVs7dLyDqjvo2MAt4NCQ+bbUKiYjElrqHRCK9QitFk8fdvWnac5mZvUCU5J/R4ueSwF2h68eA6929zsyuAn5pZq8AW4Bzw/lXA/eY2YvA08B7AO7+upn9FzAvJEINRC0rW8N1mr5gXNZO/Y8hGsvyj+G6IiJFR6s8i+yCmS0Dprj7mnzXRUSkp1P3kIiIiMSCWlpEREQkFtTSIiIiIrGgpEVERERiQUmLiIiIxIKSFhEREYkFJS0iIiISC/8PTwct3oUSq3sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1\tAverage Score: 2660.82,\tScore: 2660.82 \tTime: 00:00:18\n",
      "Episode 2\tAverage Score: 2652.16,\tScore: 2643.50 \tTime: 00:00:17\n",
      "Episode 3\tAverage Score: 2615.37,\tScore: 2541.80 \tTime: 00:00:18\n",
      "Episode 4\tAverage Score: 2623.42,\tScore: 2647.57 \tTime: 00:00:17\n",
      "Episode 5\tAverage Score: 2627.60,\tScore: 2644.33 \tTime: 00:00:17\n",
      "Episode 6\tAverage Score: 2631.54,\tScore: 2651.21 \tTime: 00:00:18\n",
      "Episode 7\tAverage Score: 2635.73,\tScore: 2660.87 \tTime: 00:00:17\n"
     ]
    }
   ],
   "source": [
    "from collections import deque\n",
    "import os\n",
    "\n",
    "def play(env, agent, n_episodes):\n",
    "    state = env.reset()\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        state = env.reset()        \n",
    "        score = 0\n",
    "        \n",
    "        time_start = time.time()\n",
    "        \n",
    "        while True:\n",
    "            action = agent.select_action(np.array(state))\n",
    "            env.render()\n",
    "            time.sleep(0.01)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break \n",
    "\n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)\n",
    "        \n",
    "        print('Episode {}\\tAverage Score: {:.2f},\\tScore: {:.2f} \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, np.mean(scores_deque), score, s//3600, s%3600//60, s%60))  \n",
    "\n",
    "play(env=env, agent=agent, n_episodes=7)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
